pubPrepCrawl

#!/usr/bin/env python

# load default python packages
import logging, optparse, os, sys, collections, gzip, re, codecs, operator, glob, random
from os.path import *
from collections import defaultdict

# add <scriptDir>/lib/ to package search path
sys.path.insert(0, join(dirname(abspath(__file__)),"lib"))

# load our own libraries
import pubConf, pubGeneric, tabfile, maxCommon, pubPubmed, pubResolvePublishers, pubCrawlConf, unidecode
import pubUpdatePmids
from urllib2 import urlparse

import xml.etree.cElementTree as etree

ISSNTAB = "issns.tab"

# ===== FUNCTIONS =======

def parseIssns(pubFname, crawlPubDirs, crawlIssnOverwrite):
    " parse Issns into dict Issn, title -> pub "
    logging.info("Parsing publisher table %s" % pubFname)
    pubs = maxCommon.iterTsvRows(pubFname)
    # parse into dict pub -> list of issns
    pubToJournals = defaultdict(list)
    for pub in pubs:
        #issns = pub.journalEIssns.split("|")
        issns = pub.journalIssns.split("|")
        titles = pub.titles.split("|")
        for issn, title in zip(issns, titles):
            pubToJournals[pub.pubName].append((issn, title))

        if len(titles)!=len(issns):
            logging.warn("Illegal fields for publisher %s, %s, %s" % (pub.pubName, pub.titles, pub.journalEIssns))
            continue
        assert(len(titles)==len(issns))
    return pubToJournals

    #logging.info("Parsing %s" % pubFname)
    #pubs = maxCommon.iterTsvRows(pubFname)
    ## parse into dict pub -> list of issns
    #pubToIssn = {}
    #issnToTitle = {}
    #for pub in pubs:
    #    issns = pub.journalEIssns.split("|")
    #    titles = pub.titles.split("|")
    #    pubToIssn[pub.pubName] = (issns, titles)
    #    if len(titles)!=len(issns):
    #        logging.warn("Illegal fields for publisher %s, %s, %s" % (pub.pubName, pub.titles, pub.journalEIssns))
    #        continue
    #    assert(len(titles)==len(issns))
    #    for title, issn in zip(titles, issns):
    #        issnToTitle[issn] = title

    ## make sure that all configured publishers  are in the dict we just read
    #issnToPub = {}
    #for pubName in crawlPubDirs:
    #    if pubName not in pubToIssn:
    #        raise Exception("publisher %s defined in pubConf not found in %s" % (pubName, pubFname))
    #    issns, titles = pubToIssn[pubName]
    #    for issn, title in zip(issns, titles):
    #        issnToPub[issn] = pubName

    ## write into dict issn -> pub
    #for corrIssn in crawlIssnOverwrite:
    #    if corrIssn in issnToPub:
    #        del issnToPub[corrIssn]

    ## now reverse again and make dict pub -> set of (issn, title)
    #pubToIssnName = {}
    #for issn, pub in issnToPub.iteritems():
    #    #if issn=="1536-4844":
    #        #print repr(issn), pub, issnToTitle.get(issn, "")
    #        #assert(False)
    #    pubToIssnName.setdefault(pub, set()).add((issn, issnToTitle.get(issn, "")))
    #return pubToIssnName

def writeIssnTables(outDir, pubIds, pubToIssn, issnOverwrite):
    """ write (issn,minYear,maxYear) as issn.tab to one publisher directories, 
        adding issnOverwrite at the end 
    """
    outFhDict = {}

    # prep the lookup table
    pubIdToDescs = {}
    for pubDesc, pubId in pubIds.iteritems():
        pubIdToDescs.setdefault(pubId, []).append(pubDesc)

    # outDir has to be identical to the pubId from pubConf
    pubId = basename(outDir)

    issnOutFname = join(outDir, ISSNTAB)

    if not isdir(outDir):
        logging.info("Creating dir %s" % outDir)
        os.makedirs(outDir)

    # lookup Issns for publisher description
    # write default ISSNs
    logging.info("Writing to %s" % issnOutFname)
    if issnOutFname not in outFhDict:
        outFh = open(issnOutFname, "w")
        outFh.write("issn\tjournal\tstartYear\tendYear\tpublisher\n")
        outFhDict[issnOutFname] = outFh
    else:
        outFh = outFhDict[issnOutFname]

    # now output ISSNs
    issnCount = 0
    for pubDesc in pubIdToDescs[pubId]:
        logging.debug("Found %d issns for publisher %s" % (len(pubToIssn[pubDesc]), pubId))
        for issn, title in pubToIssn[pubDesc]:
            logging.info("Title: %s, print ISSN %s" % (title, issn))
            if issn=="":
                continue
            line = "%(issn)s\t%(title)s\t0\t0\t%(pubDesc)s\n" % locals()
            # some journal names contain unicode, we just strip it here
            outFh.write(unidecode.unidecode(line))
            issnCount+=1

    # add manual ISSNs
    for issn, startYear, endYear in issnOverwrite.get(pubId, []):
        outFh.write("%s\t%d\t%d\t%s\n" % (issn, startYear, endYear, pubName))
        issnCount+=1
    logging.info("Wrote %d ISSNs" % issnCount)

def writeIssns(pubFname, outDir, pubDirs, issnOverwrite):
    """ use pubConf.py and the publishers.tab file to get target ISSNs
    and write them to outDir/issns.tab 
    """
    pubToIssnTitle = parseIssns(pubFname, pubDirs, issnOverwrite)
    writeIssnTables(outDir, pubDirs, pubToIssnTitle, issnOverwrite)

def writePmids(pubDir, pmids):
    " randomize pmids and write as pmids.txt to pubDir "
    random.shuffle(pmids)
    pmidFname = join(pubDir, "pmids.txt")
    logging.info("Writing %d PMIDs to %s" % (len(pmids), pmidFname))
    pmidFh = open(pmidFname, "w")
    for pmid in pmids:
        pmidFh.write(pmid+"\n")

def resolveIssnToPmid_Pubmed(issnRowFnames, minYear):
    """ use pubmed eutils to get the PMIDs for a dict of 
    pubDir -> (list of ISSN records) and append to filename """
    for pubDir, issns in issnRowFnames.iteritems():
        logging.info("Processing publisher dir %s" % pubDir)
        pmids = []
        for issnData in issns:
            # read pmids from pubmed or filesystem
            logging.info("Retrieving PMIDs for ISSN %s" % issnData.issn)
            query = issnData.issn+"[ta]"
            startYear = int(issnData.startYear)
            endYear = int(issnData.endYear)
            if startYear==0:
                startYear=str(minYear)
            if endYear==0:
                endYear="2030"
            query += " %s:%s[dp]" % (str(startYear), str(endYear))
            logging.debug("sending query to pubmed: %s" % query)
            
            issnPmids = list(pubPubmed.ncbiESearch(query, tool="pubtools_pubPrepCrawl", \
                email=pubConf.email, delaySecs=pubConf.eutilsDelaySecs))
            if len(issnPmids)==0:
                logging.warn("No PMIDs for pubmed query %s" % query)
            else:
                logging.info("Got %d PMIDs for ISSN %s" % (len(issnPmids), issnData.issn))
            pmids.extend(issnPmids)

        writePmids(pubDir, pmids)

             
def getPmidsForIssns(outDir, minYear):
    " look for issns.lst in outdir, download all PMIDs for these ISSNs and write to pmids.txt "
    #outDirFiles = glob.glob(join(outDir, "*"))
    outDirFiles = [outDir]

    # parse all issns into a big dict dirPath -> list of issns
    outDirIssns = {}
    for outDirPath in outDirFiles:
        if not isdir(outDirPath):
            logging.info("ignoring file %s (not a directory)" % outDirPath)
            continue

        #if onlyDir!=None and onlyDir!=basename(outDirPath):
            #logging.info("Ignoring %s, onlyDir option set to %s" % (outDirPath, onlyDir))
            #continue

        issnFname = join(outDirPath, ISSNTAB)
        if not isfile(issnFname):
            logging.warn("Could not find %s" % issnFname)
            continue

        logging.info("Reading %s" % issnFname)

        for row in maxCommon.iterTsvRows(issnFname):
            outDirIssns.setdefault(outDirPath, []).append(row)

    resolveIssnToPmid_Pubmed(outDirIssns, minYear)

def main(args, options):
    pubGeneric.setupLoggingOptions(options)

    command = args[0]
    if len(args)>1:
        outDir = args[1]
    minYear = options.minYear
    #onlyDir = options.onlyDir

    pubDirs = pubCrawlConf.crawlPubIds
    issnOverwrite = pubConf.crawlIssnOverwrite
    pubFname = pubConf.publisherIssnTable
    journalFname = pubConf.journalTable

    command = command.lower()

    if command=="publishers":
        journalListDir = pubConf.journalListDir
        pubResolvePublishers.initJournalDir(journalListDir, options.nlmCatalog, journalFname, pubFname)

    elif command=="issns":
        writeIssns(pubFname, outDir, pubDirs, issnOverwrite)

    elif command=="pmids":
        getPmidsForIssns(outDir, minYear)

    elif command=="allpmids":
        medlineDir = pubConf.resolveTextDir("medline")
        pubUpdatePmids.updatePmids(medlineDir, outDir, None, minYear=minYear)

    else:
        raise Exception("Unknown command %s" % command)


# === COMMAND LINE INTERFACE, OPTIONS AND HELP ===
parser = optparse.OptionParser("""usage: %prog [issns|pmids] <outDir> - prepare the directories for the crawler filling it with ISSNs and PMIDs.

outDir has to correspond to a "publisher name". Publishers are defined in
pubCrawlConf.py and need three configuration items for the crawler:
- a publisher name in the NLM Catalog / Pubmed, for %prog. This will be used by %prog to
  find the PMIDs.
- a list of rules, to specify how to crawl the files
- a delay time, between any two http requests

The journal lists for this program are stored in data/publishers/journals.tab. 
If your publisher of interest is not already recognized in the "publisher" column in 
the file journals.tab there, then the easiest way forward is probably to not
use the "issns" argument, but copy an existing issns.tab file and fill it with a handful of
ISSNs manually.

To get an idea of the problem, try this search:
    $ egrep '(^source)|(EMBO)' data/journals/journals.tab
You can see that EMBO journals sometimes show Wiley as the publisher, sometimes EMBO,
and at other times are assigned to NPG. Neither NLM, nor Wiley nor Scopus really 
show us the copyright holder.

commands:
    %prog issns <pubName>      - create a file pubName/issn.tab and fill it with all ISSNs of the 
                                 publisher named <pubName> from the file data/journals/publishers.tab

    %prog pmids <pubName>      - retrieve the PMIDs from Pubmed Eutils for all ISSNs in pubName/issns.tab
    %prog allPmids <pubName>  - retrieve the PMIDs from a local medline copy for all ISSNs 
                                in all direct *subdirectories* of baseDir. In this way, you can update
                                all PMIDs for all publishers that you crawl with a single command.

    usually not needed (the dir data/journals/ contains a pre-made list of publishers and their journals):
    %prog publishers          - re-create the list of publishers and their journals
""")

parser.add_option("-d", "--debug", dest="debug", action="store_true", help="show debug messages")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show more debug messages")
parser.add_option("", "--nlmCatalog", dest="nlmCatalog", action="store", help="use a given nlmCatalog.xml instead of the default one. Used for the 'publishers' argument.")
parser.add_option("-m", "--minYear", dest="minYear", action="store", type="int", help="minimum year for articles, default is %default", default=1990)
#parser.add_option("-o", "--onlyDir", dest="onlyDir", action="store", help="apply the PMID step only on a given directory")
(options, args) = parser.parse_args()

if len(args)<1:
    parser.print_help()
    exit(1)

main(args, options)