mklist-letterboxd-pd

#!/usr/bin/env python

"""
Extract IMDB IDs of public domain movies listed on LetterBoxd.
"""

import argparse
import json
import lxml.html
import movielib
import urllib2
import urlparse

urlbases = [
    # Films in The Public Domain
    "https://letterboxd.com/graemehalifax/list/the-public-domain/detail/page/%d/",
]

def get_movie_info(entryurl):
    try:
        root = lxml.html.fromstring(movielib.http_get_read(entryurl))
    except urllib2.HTTPError as e:
        return None
    for a in root.cssselect("p.text-link a[data-track-action=IMDb]"):
        if -1 != a.attrib['href'].find("imdb.com"):
            imdburl = a.attrib['href']
            imdburl = imdburl.replace('/maindetails', '/')
            return imdburl
    print("Unable to find IMDB id for %s" % entryurl)
    return entryurl

def get_movie_list(l, urlbase, page=1):
    summaryurl = urlbase % page
    print summaryurl
    try:
        root = lxml.html.fromstring(movielib.http_get_read(summaryurl))
    except urllib2.HTTPError as e:
        return "n/a"
    count = 0
    for div in root.cssselect("ul.film-details-list li.film-detail"):
        title = div.cssselect("h2 a")[0].text_content()
        year = div.cssselect("h2 a")[1].text_content()
        entryurl = urlparse.urljoin(summaryurl,
                                    div.cssselect("h2 a")[0].attrib['href'])
        e = get_movie_info(entryurl)
        if e:
            l[e] = {
                'status' : 'free',
                'freenessurl' : entryurl,
                'title' : title,
                'year' : year,
            }
            count = count + 1
    return count

def main():
    global urlbases
    parser = argparse.ArgumentParser()
    parser.add_argument('--baseurl', help='LetterBoxd list URL, like https://letterboxd.com/loureviews/list/internet-archive-silent-films/.')
    parser.add_argument('--output', default='free-movies-letterboxd-pd.json')
    args = parser.parse_args()

    if args.baseurl:
        if '/' != args.baseurl[-1]:
            args.baseurl = args.baseurl + '/'
        urlbases = ["%sdetail/page/%%d/" % args.baseurl]

    l = {}
    for urlbase in urlbases:
        page = 1
        while 0 < get_movie_list(l, urlbase=urlbase, page=page):
            page = page + 1
    movielib.savelist(l, args.output)

if __name__ == '__main__':
    main()