check_validation

#!/usr/bin/env python3

import urllib.request
import urllib.parse
import sys
import re
import lxml.html

# TODO: http://www.w3.org/2005/MWI/BPWG/Group/TaskForces/Checker/
# TODO: http://www.sidar.org/hera/
# TODO: YSlow
# TODO: Page Speed

# {{{ Function: retrieve_url_content ------------------------------------------
def retrieve_url_content(http_method, remote_url, params):
    """Retrieve the content of an URL that could be called in two ways (GET or
    POST) sending eventually some data.

    Keyword arguments:
    http_method    -- HTTP Method to use (GET or POST).
    remote_url     -- The remote URL that validates the URL.
    params         -- The param list, could be sent via GET or POST.

    Return value:
    a string, that contain the URL response."""

    if (http_method == 'GET'):
        data = urllib.request.urlopen(remote_url).read()
    else:
        data = urllib.request.urlopen(remote_url, params).read()

    return data
# }}} -------------------------------------------------------------------------

# {{{ Function: validate ------------------------------------------------------
def validate(http_method, remote_url, params, page, match,
             excepted_value='.*', match_reverse=False):
    """Validate an URL calling the defined web site validator,
    passing some data to it.

    Keyword arguments:
    http_method    -- HTTP Method to use (GET or POST).
    remote_url     -- The remote URL that validates the URL.
    params         -- The param list, could be sent via GET or POST.
    page           -- The current page that needs validation.
    match          -- The RegExp to match the "excepted_value" inside the HTML
                      response.
    excepted_value -- The excepted value to consider an answer good or not.
    match_reverse  -- The flag that means "if the excepted value is not
                      matched". This behavious is useful to exclude from the
                      match, for example, an error string.

    Return value:
    a string, that contain the response of validation."""

    page   = urllib.request.quote(page)
    params = params.replace('%25URI%25', page).replace('%URI%', page)

    if (http_method == 'GET'):
        remote_url = remote_url + '%s' % params

    data = retrieve_url_content(http_method, remote_url, params)

    tree = lxml.html.fromstring(data)

    elements = tree.xpath(match)
    text     = ''.join(elements)
    match    = re.match(excepted_value, text, re.DOTALL)

    status = (match != None)

    if (match_reverse):
        status = not status

    if status:
        return 'OK'
    else:
        return 'FAIL' + ' > Check this out to: ' + remote_url
# }}} -------------------------------------------------------------------------

# Main ------------------------------------------------------------------------
if __name__ == "__main__":
    print('CHECK VALIDATION')
    
    v = sys.argv[1]
    
    ###############################################################################
    # Web Site Validation of: Homepage
    ###############################################################################
    print('Validate W3C:')
    
    url_param = 'uri=%URI%&charset=(detect automatically)&doctype=Inline'
    url_param += '&group=0&user-agent=W3C_Validator/1.2'
    url_param = urllib.parse.quote(url_param)
    xpath = './/*[@id="form"]//*[text()="Result:"]/'
    xpath += 'following-sibling::*[1]/text()'
    
    res = validate('GET', 'http://validator.w3.org/check?', url_param, v,
                   xpath, '.*Passed.*')
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: CSS v3
    ###############################################################################
    print('Validate CSS 3:')
    
    url_param = 'uri=%URI%&profile=css3&usermedium=all&warning=2&vextwarning=&'
    url_param += 'lang=en'
    xpath = './/*[@id="results_container"]//*[local-name()="div"]/'
    xpath += '*[local-name()="h3"]/text()'
    
    res = validate('GET', 'http://jigsaw.w3.org/css-validator/validator?',
                   url_param, v, xpath,
                   '.*Congratulations! No Error Found\..*')
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: CSS v2.1
    ###############################################################################
    print('Validate CSS 2.1:')
    
    url_param = 'uri=%URI%&profile=css21&usermedium=all&warning=2&vextwarning='
    url_param += '&lang=en'
    xpath = './/*[@id="results_container"]//*[local-name()="div"]/'
    xpath += '*[local-name()="h3"]/text()'
    
    res = validate('GET', 'http://jigsaw.w3.org/css-validator/validator?',
                   url_param, v, xpath,
                   '.*Congratulations! No Error Found\..*')
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: CSS v2
    ###############################################################################
    print('Validate CSS 2:')
    
    url_param = 'uri=%URI%&profile=css2&usermedium=all&warning=2&vextwarning=&'
    url_param += 'lang=en'
    xpath = './/*[@id="results_container"]//*[local-name()="div"]/'
    xpath += '*[local-name()="h3"]/text()'
    
    res = validate('GET', 'http://jigsaw.w3.org/css-validator/validator?',
                   url_param, v, xpath,
                   '.*Congratulations! No Error Found\..*')
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: CSS vMobile
    ###############################################################################
    print('Validate CSS Mobile:')
    
    url_param = 'uri=%URI%&profile=mobile&usermedium=all&warning=2'
    url_param += 'vextwarning=&lang=en'
    xpath = './/*[@id="results_container"]//*[local-name()="div"]/'
    xpath += '*[local-name()="h3"]/text()'
    
    res = validate('GET', 'http://jigsaw.w3.org/css-validator/validator?',
                   url_param, v, xpath,
                   '.*Congratulations! No Error Found\..*')
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: Feed
    ###############################################################################
    print('Validate Feed:')
    
    url_param = urllib.parse.urlencode({'url': '%URI%'})
    xpath = './/*[@id="main"]//*[local-name()="h2"]/text()'
    
    res = validate('GET', 'http://validator.w3.org/feed/check.cgi?',
                   url_param, v, xpath, '.*Congratulations!.*')
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: HTTP Header
    ###############################################################################
    print('Validate HTTP Headers:')
    
    url_param = urllib.parse.urlencode({'uri': '%URI%'})
    xpath = './/*[@class="bad msg"]/text()'
    
    res = validate('GET', 'http://redbot.org/?', url_param, v, xpath, '.+',
                   True)
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: Semantics
    ###############################################################################
    print('Validate Semantics:')
    
    url_param = urllib.parse.urlencode({'url': '%URI%', 'html': ''})
    
    xpath = './/*[@id="extracted-data-google"]/*[local-name()="div"][2]//text()'
    
    res = validate('GET',
                   'http://www.google.com/webmasters/tools/richsnippets?',
                   url_param, v, xpath, '.*No data detected.*', True)
    
    print(res)
    
    ###############################################################################
    # Web Site Validation of: Links
    ###############################################################################
    print('Validate Links:')
    
    url_param = 'uri=%URI%&summary=on&hide_type=all&depth=&check=Check'
    xpath = './/*[@id="main"]//text()'
    
    res = validate('GET', 'http://validator.w3.org/checklink?', url_param,
                   v, xpath, '.*Valid anchors.*')
    
    print(res)