ted_talk_sentiment.py

import re
import os
import cPickle as cp
import numpy as np
from itertools import product
from scipy.stats import ttest_ind
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import nltk
nltk.download('punkt')
from nltk.tokenize import sent_tokenize
from bluemix import parse_sentence_tone

from TED_data_location import ted_data_path

############################### Generic Readers ##############################
'''
These functions (generic readers) go to the input of sentiment comparator
class as preset values of the "reader" variable. All these functions follow
the same input-output convention. The input is the fullpath of a pickle 
file containing the talk transcript and meta data. The output is a list
containing the transcripts in a specific format
'''
#def read_sentences(pklfile):
#    '''
#    Read talk transcripts and tokenize the sentences
#    While tokenizing, it removes sound tags: (Applause), (Laughter) etc.    
#    '''
#    assert os.path.isfile(pklfile),'File not found: '+pklfile
#    data = cp.load(open(pklfile))
#    txt = re.sub('\([a-zA-Z]*?\)','',' '.join([anitem for apara in \
#        data['talk_transcript'] for anitem in apara]))
#    return sent_tokenize(txt)

def read_bluemix(pklfile,sentiment_dir='TED_feature_bluemix_scores/'):
    '''
    Reads all the sentences and their corresponding bluemix sentiments.
    Note: DONOT change the name of this function. It is used somewhere
    else in the code
    '''
    pklpath = os.path.join(ted_data_path,sentiment_dir)
    pklfile = os.path.join(pklpath,str(pklfile)+'.pkl')
    assert os.path.isfile(pklfile),'File not found: '+pklfile
    data = cp.load(open(pklfile))
    assert data.get('sentences_tone'), \
        'Sentence-wise sentiment is not available: {0}'.format(pklfile)
    scores,header,sentences,_,_ = parse_sentence_tone(data['sentences_tone'])
    return scores,header,sentences
########################### End of Generic Readers ###########################

######################### Sentiment Extractors ###############################
# These functions go to the input of sentiment comparator class as preset
# values of the variable "extractor". By default, use vadersentiment.
# the functions take a sentence to calculate sentiment and outputs two lists.
# The first one is a list of sentiments, the second list is the names of the
# corresponding sentiment.
#analyzer = SentimentIntensityAnalyzer()
#def vadersentiment(asent):
#    results = analyzer.polarity_scores(asent)
#    return [results['neg'],results['neu'],results['pos'],results['compound']],\
#    ['negative','neutral','positive','compound']
##############################################################################

class Sentiment_Comparator(object):
    '''
    Sentiment Comparator class contains all the information to compare
    sentiment data between several groups of files.
    It performs the following major tasks. Given the ids of talks for a number of
    groups, it extracts the raw sentiments values for each sentence or utterances
    of a talk, smoothens the time-series formed by those raw sentiment values and
    then interpolates the time-series to a common length (100 sample). This data
    is preserved within the class so that it can be possible to see the time
    averages or ensemble averages of the time-series. It also stores the backward
    references to the original sentences after the interpolation, so that it is
    possible to find from which sentences the interpolated samples are derived.

    It contains the following inputs and variables:
    groups        : Input dictionary with group name as keys and the talk indices
                    as values
    raw_sentiments: A dictionary storing the raw sentiments.
                    The keys are the talk ids and values
                    are matrices for which columns represent sentiment as
                    mentioned in the column_names variable
    column_names  : Name of columns of the sentiment matrix. For vadersentiment
                    it is ['neg','neu','pos']. This variable is populated
                    when loading the raw sentiments
    sentiments_interp: The sentiment data interpolated to a canonical size
    back_ref      : Reference to the old indices of the sentences from 
                    each interpolated sample
    alltalks      : List of all the talk ids being analyzed in this comparator
    '''
    def __init__(self,
                dict_groups,
                process=True):
        self.groups = dict_groups
        self.alltalks = [ids for agroup in self.groups \
            for ids in self.groups[agroup]]
        
        self.raw_sentiments = {}
        self.sentiments_interp={}
        self.back_ref={}
        self.column_names=[]
        if process:
            self.extract_raw_sentiment()
            self.smoothen_raw_sentiment()
            self.intep_sentiment_series()

    # Fill out self.raw_sentiments
    def extract_raw_sentiment(self):
        for i,atalk in enumerate(self.alltalks):
            scores,header,_ = read_bluemix(atalk,'TED_feature_bluemix_scores/')
            if i==0:
                self.column_names = header
            self.raw_sentiments[atalk] = scores

    def reform_groups(self,new_dict_groups):
        '''
        If it becomes necessary to re-group the data, this method
        comes handy. It will restructure the object without loading
        the data again from file (loading the data is time cosuming).
        Note that the new_dict_groups dictionary must
        contain all the talkids from the original dict_groups. No more,
        no less. Only the group assignments are meant to be changed.
        '''
        self.groups = new_dict_groups
        self.raw_sentiments = {atalk:self.raw_sentiments[atalk] \
            for akey in new_dict_groups \
                for atalk in new_dict_groups[akey]}
        self.sentiments_interp = {atalk:self.sentiments_interp[atalk] \
            for akey in new_dict_groups \
                for atalk in new_dict_groups[akey] }

    # Changes the self.raw_sentiments to a smoothed version
    def smoothen_raw_sentiment(self,kernelLen=5):
        # Get number of columns in sentiment matrix 
        _,n = np.shape(self.raw_sentiments[self.alltalks[0]])

        for atalk in self.alltalks:
            temp=[]
            for i in range(n):
                temp.append(np.convolve(\
                self.raw_sentiments[atalk][:,i],\
                np.ones(kernelLen)/float(kernelLen),mode='valid'))
            self.raw_sentiments[atalk]=np.array(temp).T

    def intep_sentiment_series(self,bins=100):
        '''
        Fills out the variable self.sentiments_interp. Different sentiment
        series has different lengths due to the variable length of the talks.
        This function brings all the series in a common length (having 
        100 samples). It also updates the backward reference (back_ref)
        '''        
        for atalk in self.alltalks:
            m,n = np.shape(self.raw_sentiments[atalk])
            # Pre-allocate
            self.sentiments_interp[atalk] = np.zeros((bins,n))
            # x values for the interpolation
            old_xvals = np.arange(m)
            new_xvals = np.linspace(0,old_xvals[-1],num=bins)
            # Update the backward reference
            self.back_ref[atalk] = [np.where((old_xvals>=lo) & \
                (old_xvals<=hi))[0].tolist() for lo,hi in \
                zip(new_xvals[:-1],new_xvals[1:])]+[[old_xvals[-1]]]
            # Interpolate column by column
            for i in range(n):
                self.sentiments_interp[atalk][:,i] = \
                np.interp(new_xvals,old_xvals,self.raw_sentiments[atalk][:,i])

    # Calculates (and returns) the ensemble averages of the groups
    def calc_group_mean(self):
        group_average = {}
        for agroup in self.groups:
            vals = [self.sentiments_interp[id] for id in self.groups[agroup]]
            # Averaging over the talks in a group
            group_average[agroup]=np.mean(vals,axis=0)
        return group_average

    def calc_time_mean(self,perform_ttest=True):
        '''
        Calculates (and returns) the Time averages of the sentiments
        Also returns the p-values if ttest is done. Note: ttest can't
        be done for more than 2 groups
        '''
        time_avg = {}
        for agroup in self.groups:
            vals = [self.sentiments_interp[id] for id in self.groups[agroup]]
            # Averaging over time
            time_avg[agroup]=np.mean(vals,axis=1)
        # Perform ttest for statistical significance
        if perform_ttest:
            pvals=[]
            m,n = np.shape(time_avg[agroup])
            allkeys=time_avg.keys()
            assert len(allkeys)==2,'T-test can not be done for 2+ groups'
            for i in range(n):
                print 'Sentiment:',self.column_names[i],
                _,p = ttest_ind(time_avg[allkeys[0]][:,i],
                    time_avg[allkeys[1]][:,i])
                print 'p-value:',p
                pvals.append(p)
        # Average and return
        for agroup in time_avg:
            time_avg[agroup]=np.mean(time_avg[agroup],axis=0)
        if not perform_ttest:
            return time_avg
        else:
            return time_avg,pvals

    def display_sentences(self,talkid,start_percent,
        end_percent,selected_columns = None):
        '''
        Even though the sentiment plots are interpolated to 0 to 100
        to calculate ensemble averages, the reference to the original
        sentence is not lost. Every talk keeps a "backword reference"
        indicating what are the original sentences (actually the sentence number)
        '''
        assert talkid in self.back_ref.keys(),\
            'The specified talkid is not present in the talklist'
        sent_ind=[]
        _,_,sentences = read_bluemix(talkid,'TED_feature_bluemix_scores/')
        # Print the desired ones
        print 'talkid:',talkid
        print '================='
        for i in range(start_percent,end_percent):
            for j in self.back_ref[talkid][i]:
                print 'percent:',i,'sentence#:',j,sentences[j]
                print 'Sentiment Rating:',
                if not selected_columns:
                    for k in range(np.size(self.raw_sentiments[talkid],axis=1)):
                        print self.column_names[k],self.raw_sentiments[talkid][j,k]
                else:
                    for k in selected_columns:
                        print self.column_names[k],self.raw_sentiments[talkid][j,k]
                print
                print

################################ Plotters ####################################
def draw_single_sentiment(sentim_scores,
                          column_names,
                          selected_columns = None,
                          full_y=False,
                          legend_location='lower center',
                          outfilename=None):
    '''
    # Draws the sentiment values of a single talk. The input array
    # can be either raw sentiments or interpolated sentiments
    '''
    plt.figure(figsize=(16, 4))
    if not selected_columns:
        plt.plot(sentim_scores,label=column_names)
    else:
        for col in selected_columns:
            plt.plot(sentim_scores[:,col],label=column_names[col])
    plt.xlabel('Sentence Number')
    plt.ylabel('Values')
    plt.subplots_adjust(bottom=0.14, right=0.99, left=0.05, top=0.78)
    plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
           ncol=5, mode="expand", borderaxespad=0.)
    if full_y:
        plt.ylim([0,1])
    #plt.tight_layout()
    if outfilename:
        plt.savefig(outfilename)
    else:
        plt.show()

def draw_group_means(avg,column_names,fullyaxis=False,outfilename=None):
    '''
    Draw the averages of various groups. It creates as many figures
    as the seniment scores (13 for Bluemix). The difference between this
    function and draw_group_mean_sentiments function is: this one groups all
    the plots corresponding to a single sentiment score together in one 
    figure. where as the other one groups all the plots corresponding
    to a single group (cluster) together in one figure.
    '''
    m,n = avg[avg.keys()[0]].shape
    # Draw the group averages
    for acol in range(n):
        plt.figure(figsize=(16,9))
        for akey in avg:
            plt.plot(avg[akey][:,acol],label=akey)
        plt.xlabel('Percent of Talk Progression')
        plt.ylabel('Value')
        if fullyaxis:
            plt.ylim([0,1])
        plt.title(column_names[acol])
        plt.tight_layout()
        plt.subplots_adjust(bottom=0.05, right=0.99, left=0.05, top=0.85)
        plt.legend(bbox_to_anchor=(0., 1.05, 1., 0), loc=3,\
           ncol=5, mode="expand", borderaxespad=0.)
        if outfilename:
            split_fn = os.path.split(outfilename)
            plt.savefig(os.path.join(split_fn[0],column_names[acol]+\
                '_'+split_fn[1]))
    if not outfilename:
        plt.show()

def draw_group_mean_sentiments(grp_means,
                            column_names,
                            selected_columns=None,
                            styles=['r.','r--','r-','r.-',
                                    'b.','b--','b-','b.-'],
                            legend_location='lower center',
                            outfilename=None):
    '''
    Draws the ensemble averages of the sentiments. Check the note on
    difference in draw_group_mean function documentation.
    '''
    plt.figure(figsize=(16, 8))    
    for g,agroup in enumerate(grp_means):
        m,n = np.shape(grp_means[agroup])
        if not selected_columns:
            # Columns are not explicitely selected, show as much as it can
            for col in range(n):
                pltLabel = str(agroup+'_'+column_names[col])
                pltLabel = pltLabel.lower()
                pltLabel = re.sub(\
                    'percent|social|tone|language|big5|range',\
                    '',pltLabel)
                pltLabel = re.sub(r'emotion_','',pltLabel)
                
                pltLabel = re.sub(r'_+',' ',pltLabel)
                if len(styles)>=n:
                    plt.plot(grp_means[agroup][:,col],
                        styles[g*len(column_names)+col],
                        label=pltLabel)
                else:
                    plt.plot(grp_means[agroup][:,col],
                        label=pltLabel)
        else:
            # Columns are explicitely selected. Show only those            
            for i,col in enumerate(selected_columns):
                pltLabel = str(agroup+'_'+column_names[col])                
                pltLabel = pltLabel.lower()
                pltLabel = re.sub(\
                    'percent|social|tone|language|big5|range',\
                    '',pltLabel)
                pltLabel = re.sub('emotion_','',pltLabel)
                pltLabel = re.sub('_+',' ',pltLabel)
                plt.plot(grp_means[agroup][:,col],
                        styles[g*len(selected_columns)+i],
                        label=pltLabel)
        plt.ylabel('Values')
        plt.subplots_adjust(bottom=0.05, right=0.99, left=0.05, top=0.88)
        plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,\
           ncol=5, mode="expand", borderaxespad=0.)
    if outfilename:
        plt.savefig(outfilename)
    else:
        plt.show()

# Draw bar plots for time averages and annotate pvalues
def draw_time_mean_sentiments(time_avg,
                            column_names,
                            pvals,
                            groupcolor=['royalblue','darkkhaki'],
                            outfilename=None):
    plt.figure(figsize=(6,6))
    keylist=sorted(time_avg.keys())    
    for i,grp in enumerate(keylist):
        print grp,':',time_avg[grp]
        plt.barh(bottom=np.arange(len(time_avg[grp]))-i*0.25+1.125,
                width=time_avg[grp],
                color=groupcolor[i],
                height= 0.25)

    plt.xlabel('Scores')
    plt.yticks(np.arange(len(time_avg[grp]))+1,\
        [namefix(c)+'\n p='+'{0:.4f}'.format(p) for c,p in \
        zip(column_names,pvals)])
    for i,ap in enumerate(pvals):
        if ap<0.01:
            topofbar = max([time_avg[grp][i] for grp in time_avg])
            plt.text(topofbar+0.005,i+0.75,'**',fontsize=18)
        elif ap<0.05:
            topofbar = max([time_avg[grp][i] for grp in time_avg])
            plt.text(topofbar+0.005,i+0.75,'*',fontsize=18)


    # plt.legend(loc='upper left',ncol=2)
    axpos=plt.gca().get_position()    
    plt.legend(keylist,bbox_to_anchor=(-0.3, 1.02, 1.3, .102), loc=3,
           ncol=2, mode="expand", borderaxespad=0.)
    # ax = plt.gca()
    # ax.set_xticks(np.arange(len(time_avg[grp]))+1)
    # ax.set_xticklabels(
    #     [namefix(c)+'\n p='+'{0:.4f}'.format(p) for c,p in \
    #     zip(column_names,pvals)],rotation=60)
    plt.tight_layout()
    plt.subplots_adjust(top=0.9)
    if outfilename:
        plt.savefig(outfilename)
    else:
        plt.show()

def namefix(astr):
    astr=astr.replace('emotion_tone_','').replace('language_tone_','')
    astr=astr.replace('social_tone_','').replace('_big5','')
    return astr