[IMP] oca_publish_modules: Improve flow

setup.py

@@ -35,6 +35,7 @@
         'polib',
         'pygments',
         'requests',
+        'selenium',
         'twine',
         'wheel',
     ],
@@ -62,6 +63,7 @@
             'oca-migrate-branch = tools.migrate_branch:main',
             'oca-migrate-branch-empty = tools.migrate_branch_empty:main',
             'oca-main-branch-bot = tools.main_branch_bot:main',
+            'oca-publish-modules = tools.publish_modules:main',
             'oca-pypi-upload = tools.pypi_upload:cli',
             'oca-dist-to-simple-index = tools.dist_to_simple_index:main',
             'oca-gen-addon-readme = tools.gen_addon_readme:gen_addon_readme',

tools/config.py

@@ -16,6 +16,13 @@ def init_config():
     config.add_section("odoo")
     config.set("odoo", "username", "")
     config.set("odoo", "password", "")
+    config.add_section("apps.odoo.com")
+    config.set("apps.odoo.com", "username", "")
+    config.set("apps.odoo.com", "password", "")
+    config.set(
+        "apps.odoo.com", "chromedriver_path",
+        "/usr/lib/chromium-browser/chromedriver",
+    )
     write_config(config)
 
 

tools/publish_modules.py

@@ -0,0 +1,234 @@
+# Copyright 2019 Brainbean Apps (https://brainbeanapps.com)
+# Copyright 2019 Tecnativa - Pedro M. Baeza
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+"""
+This script helps you to add all the OCA repositories in batch in apps.odoo.com
+platform. For now it's not adapted to other organization as it takes the
+information from static mapping here. It shouldn't be too much difficult to
+adapt this to other organizations.
+
+It also serves as status scanner for knowing which repos are not yet being
+automatically scanned (showing the conflict but ommited in case of simply
+empty repo), and allowing to force the scan in that moment. Note that current
+platform auto-scan all the repositories daily, so this operation is not really
+needed except being in a hurry.
+
+WARNING: This work is based on current platform implementation. This might
+stop working if Odoo changes it through time.
+
+Installation
+============
+
+It requires to have Python library `selenium`, that you can install regularly
+through pip doing `sudo pip3 install selenium` or using specific OS packages.
+
+It also requires chromedriver binary. If you are in Ubuntu or derivative, you
+can do:
+
+`sudo apt-get install chromium-chromedriver`
+
+You would need to enter afterwards the path for this binary field. Installing
+the package above, the path is /usr/lib/chromium-browser/chromedriver.
+
+Configuration
+=============
+
+You can have a file called oca.cfg on the same folder of the script for
+storing credentials parameters. You can generate an skeleton config running
+this script for a first time.
+
+The credentials are stored in the section [apps.odoo.com], with the names
+"username", "password" and "chromedriver_path", which are self-explanatory.
+
+If not set, a prompt will ask you to enter user and password. You would also
+need to specify through an environment variable called "CHROMEDRIVER" the
+chromedriver path if not specified in the config file.
+
+Usage
+=====
+
+oca-publish-modules [OPTIONS]
+
+Options:
+  --branch TEXT                   Limit to specific Odoo series. eg 11.0.
+  --repository TEXT               Limit to a repository. eg contract.
+  --registration / --no-registration
+                                  Perform the registration of repositories.
+  --status / --no-status          Retrieve the status of bad repositories.
+  --force-scan / --no-force-scan  If auto-scan not activated, activate it and
+                                  perform a scan in that moment.
+  --scan-skip-empty / --scan-no-skip-empty
+                                  Skip scan of empty repositories (no matter
+                                  force scan value).
+  --help                          Show the help.
+"""
+
+import os
+import logging
+from getpass import getpass
+import click
+from selenium.common.exceptions import NoSuchElementException
+from selenium import webdriver
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.chrome.options import Options
+from __future__ import print_function
+
+from .oca_projects import get_repositories_and_branches, url
+from .config import read_config
+
+_logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option('--branch', 'target_branch',
+              help="Limit to specific Odoo series. eg 11.0.")
+@click.option('--repository', 'target_repository',
+              help="Limit to a repository. eg contract.")
+@click.option('--registration/--no-registration', 'do_registration',
+              default=True,
+              help="Perform the registration of repositories.")
+@click.option('--status/--no-status', 'do_status',
+              default=True,
+              help="Retrieve the status of bad repositories.")
+@click.option('--force-scan/--no-force-scan', 'force_scan',
+              default=False,
+              help="If auto-scan not activated, activate it and perform a "
+                   "scan in that moment.")
+@click.option('--scan-skip-empty/--scan-no-skip-empty', 'scan_skip_empty',
+              default=True,
+              help="Skip scan of empty repositories (no matter force scan "
+                   "value).")
+def main(target_branch, target_repository, do_registration, do_status,
+         force_scan, scan_skip_empty):
+    config = read_config()
+    chrome = config.get('apps.odoo.com', 'chromedriver_path')
+    if not chrome:
+        chrome = os.getenv('CHROMEDRIVER')
+    if not chrome:
+        print(
+            'Please specify CHROMEDRIVER environment variable or config'
+            ' option pointing to a valid Google Chrome Driver executable file'
+            ' and ensure its location is available via PATH environment'
+            ' variable.\n\nSee http://chromedriver.chromium.org/downloads'
+        )
+        return
+    user = config.get('apps.odoo.com', 'username')
+    if not user:
+        user = input('Odoo.com publisher account:')
+    password = config.get('apps.odoo.com', 'password')
+    if not password:
+        password = getpass(prompt='Odoo.com account password:')
+    # Selenium options
+    options = Options()
+    options.headless = True
+    driver = webdriver.Chrome(
+        executable_path=chrome,
+        options=options,
+    )
+    login(driver, user, password)
+    # First pass: register all repositories (if already registered, there
+    # will be an immediate warning in the current browser page and won't
+    # continue, so we can simply overwrite the value in the field).
+    if do_registration:
+        for repository, branch in get_repositories_and_branches():
+            if target_branch and branch != target_branch:
+                continue
+            if target_repository and target_repository != repository:
+                continue
+            repository_url = url(repository) + '#' + branch
+            print('INFO: Adding %s#%s from %s... (if not yet present)' % (
+                repository,
+                branch,
+                repository_url,
+            ))
+            register_repository(driver, repository_url)
+    # Second pass: check published state and try to publish if not yet done
+    if do_status:
+        driver.get('https://apps.odoo.com/apps/dashboard/repos')
+        for repository, branch in get_repositories_and_branches():
+            if target_branch and branch != target_branch:
+                continue
+            if target_repository and target_repository != repository:
+                continue
+            repository_url = url(repository) + '#' + branch
+            scan_repository(
+                driver, repository_url, force_scan, scan_skip_empty,
+            )
+
+
+def login(driver, user, password):
+    wait = WebDriverWait(driver, 10)
+    driver.get(
+        'https://www.odoo.com/web/login?redirect=%2Foauth2%2Fauth%2F%3Fscope'
+        '%3Duserinfo%26redirect_uri%3Dhttps%253A%252F%252Fapps.odoo.com%252F'
+        'auth_oauth%252Fsignin%26state%3D%257B%2522p%2522%253A%2B1%252C%2B'
+        '%2522r%2522%253A%2B%2522%25252F%25252Fapps.odoo.com%25252Fapps%25'
+        '22%252C%2B%2522d%2522%253A%2B%2522apps%2522%257D%26response_type%3D'
+        'token%26client_id%3Da0a30d16-6095-11e2-9c70-002590a17fd8&scope=user'
+        'info&mode=login&redirect_hostname=https%3A%2F%2Fapps.odoo.com&login='
+    )
+    login_field = driver.find_element_by_id('login')
+    login_field.clear()
+    login_field.send_keys(user)
+    password_field = driver.find_element_by_id('password')
+    password_field.clear()
+    password_field.send_keys(password)
+    login_button = driver.find_element_by_xpath(
+        './/form[@action="/web/login"]//button[@type="submit"]'
+    )
+    login_button.click()
+    wait.until(
+        lambda driver: driver.current_url == 'https://apps.odoo.com/apps'
+    )
+
+
+def register_repository(driver, repository):
+    driver.get('https://apps.odoo.com/apps/upload')
+    url_field = driver.find_element_by_name('url')
+    url_field.clear()
+    url_field.send_keys(repository)
+    submit_button = driver.find_element_by_id('apps_submit_repo_button')
+    submit_button.click()
+
+
+def scan_repository(driver, repository_url, force_scan, scan_skip_empty):
+    wait = WebDriverWait(driver, 300)
+    try:
+        item_container = driver.find_element_by_xpath(
+            './/span[@id="repo_url" and text()="%s"]'
+            '/ancestor::li[1]' % repository_url
+        )
+    except NoSuchElementException:
+        print("WARNING: %s not registered in this account." % repository_url)
+        return
+    try:
+        error_item = item_container.find_element_by_xpath(
+            './/div[@id="help_error"]',
+        )
+    except NoSuchElementException:
+        error_item = False
+    is_empty = False
+    if error_item:
+        error_text = error_item.find_element_by_xpath('.//p')
+        is_empty = error_text.text.startswith('No module found in repository')
+        if not is_empty:
+            print("ERROR: %s:\n%s" % (repository_url, error_text.text))
+    if force_scan and not (is_empty and scan_skip_empty):
+        print("INFO: Doing the scan on %s" % repository_url)
+        auto_scan_checkbox = item_container.find_element_by_xpath(
+            './/input[@name="auto_scan"]'
+        )
+        if not auto_scan_checkbox.is_selected():
+            auto_scan_checkbox.click()
+            scan_link = item_container.find_element_by_class_name(
+                'js_repo_scan'
+            )
+            scan_link.click()
+            wait.until(
+                lambda driver: driver.current_url
+                == 'https://apps.odoo.com/apps/dashboard/repos'
+            )
+
+
+if __name__ == '__main__':
+    main()