Tracking Social Issues and Topics in Presidential Speeches

 
# import required libraries to scrape presidential transcripts
from bs4 import BeautifulSoup
import pandas as pd
import pickle
import urllib2
import re

 
def get_speech_links():
    '''
    scrape content of pages with all presidential transcript links
    '''
    home_url = 'http://millercenter.org/president/speeches'
    try:
        response = urllib2.urlopen(home_url)
        page_source = response.read()
        soup = BeautifulSoup(page_source, "html5lib")
        transcript_links = soup.findAll("a", {'class': 'transcript'})
        return transcript_links
    except urllib2.HTTPError:
        print 'Homepage not available!'
        return None

 
def get_transcript(speech_link):
    '''
    scrape title of speech, date of speech and full transcipt
    contained in the input speech_link URL
    '''
    speaking = speech_link.split('/')[2]
    new_link = base_url + str(speech_link)
    try:
        response = urllib2.urlopen(new_link)
        page_source = response.read()
        soup = BeautifulSoup(page_source, "html5lib")
        title = soup.find('title').text
        speech_date = title.split('(', 1)[1].split(')')[0]
        transcript = soup.find('div', {'id': 'transcript'}).text
        transcript = transcript.replace('\n', ' ').replace('\r', '').replace('\t', '')
        return {'speaker': speaking,
                'date': speech_date,
                'title': title,
                'transcript': transcript}
    except urllib2.HTTPError:
        print 'skipped ' + str(speech_link)
        return None

# iterate through all links and extract content
transcript_links = get_speech_links()
base_url = 'http://millercenter.org/'
transcript_dict = {}
for i, link in enumerate(transcript_links):
    if i % 100 == 0:
        print 'Scraped ' + str(i) + '/' + str(len(transcript_links)) + ' of links...'
    if link.has_attr('href'):
        transcript_data = get_transcript(link['href'])
        if transcript_data is not None:
            key = transcript_data['speaker'] + '|' + transcript_data['date']
            transcript_dict[key] = transcript_data

 
# dump dataframe to pickle object
df = pd.DataFrame.from_dict(transcript_dict, orient='index')
pickle.dump(df, open( "presidential_speeches.pickle", "wb" ))

 
%matplotlib inline  
from gensim.parsing.preprocessing import STOPWORDS
from gensim.utils import simple_preprocess
from gensim.models import TfidfModel, LsiModel
from gensim.models.ldamodel import LdaModel
from gensim import corpora
from gensim import matutils
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from collections import defaultdict
import pyLDAvis.gensim as gensimvis
import pyLDAvis
import pandas as pd
import numpy as np
import pickle
import lda

 
def tokenize(text):
    return [token for token in simple_preprocess(text) if token not in STOPWORDS]

 
df = pickle.load( open( "presidential_speeches.pickle", "rb" ) )
speeches = df.transcript
documents = speeches.tolist()
texts = [tokenize(document) for document in documents]

 
frequency = defaultdict(int)
for text in texts:
    for token in text:
        frequency[token] += 1

 
texts = [[token for token in text if frequency[token] > 10] for text in texts]

 
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]

# fit LDA model
speeches_topics = LdaModel(corpus=corpus,
                           id2word=dictionary,
                           num_topics=20,
                           passes=10)

# print out first 10 topics
for i, topic in enumerate(speeches_topics.print_topics(10)):
    print '{} --- {}'.format(i, topic)

 
vis_data = gensimvis.prepare(speeches_topics, corpus, dictionary)
pyLDAvis.display(vis_data)

 
# example command to print out main topics inferred in a presidential speech
print speeches_topics.get_document_topics(corpus[10]) 

 
# extract all document-topic distritbutions to dictionnary
document_key = list(speeches.index)
document_topic = {}
for doc_id in range(len(corpus)):
    docbok = corpus[doc_id]
    doc_topics = speeches_topics.get_document_topics(docbok, 0)
    tmp = []
    for topic_id, topic_prob in doc_topics:
        tmp.append(topic_prob)
    document_topic[document_key[doc_id]] = tmp

 
# convert dictionnary of document-topic distritbutions to dataframe
df = pd.DataFrame.from_dict(document_topic, orient='index')

 
president_speech = [x.split('|')[0] for x in df.index]
year_speech = [y.split(',')[1] if ',' in y else None for y in [x.split('|')[1] for x in df.index]]

 
topics_speech = df
topics_speech['president'] = pd.Series(president_speech, index=df.index)
topics_speech['year'] = pd.Series(year_speech, index=df.index)
topic_column_names = ['topic_' + str(i) for i in range(0, 20)]
topic_column_names.append('president')
topic_column_names.append('year')
topics_speech.columns = topic_column_names
print topics_speech.head(5)
# topics_speech.to_csv('topics_by_speech.csv', sep=',')

 
columns = ['topic_'+str(i) for i in range(0, 20)] # define columns to process
df = pd.DataFrame(topics_speech.groupby('year')[columns].sum()) # group topics frequency by year
df = 100 * df.div(df.sum(axis=1), axis=0) # normalize topic frequencies by year
df = np.round(df, 1) # round topic frequencies
df.to_csv('topics_by_year.csv', sep=',')

 
import rpy2

 
%load_ext rpy2.ipython

 
%%R
library(htmlwidgets)
library(dygraphs)
library(zoo)
library(xts)

dt <- read.zoo("topics_by_year.csv", sep = ",", index.column = 1, header = TRUE, format = "%Y")
dt_large <- dt[, which(apply(dt, 2, sum)>=500)]
dt_large$others <- 100- apply(dt_large, 1, sum)
dt_large <- as.xts(dt_large)

 
%%R
dygraph(dt_large) %>%
dyAnnotation("1970-1-1", text = "Vietnam") %>%
  dyShading(from = "1961-1-1", to = "1975-1-1", color = "#bdbdbd")  %>%
  dyShading(from = "1938-1-1", to = "1945-1-1", color = "#bdbdbd")  %>%
  dyShading(from = "1914-1-1", to = "1919-1-1", color = "#bdbdbd")  %>%
  dyShading(from = "1861-1-1", to = "1865-1-1", color = "#bdbdbd")  %>%
  dyRoller(rollPeriod = 8) %>%
  dyOptions(stackedGraph = TRUE) %>%
  dyRangeSelector(height = 20) %>%
  dyLegend(show = "follow") %>%
  #dyLegend(width = 900) %>%
  dySeries("topic_1", label = "(sailors/chili)") %>%
  dySeries("topic_4", label = "(world/countries)") %>%
  dySeries("topic_6", label = "(law/congress)") %>%
  dySeries("topic_7", label = "(bank/silver/bank)") %>%
  dySeries("topic_10", label = "(work/people/children)") %>%
  dySeries("topic_12", label = "(peace/war)") %>%
  dySeries("topic_13", label = "(law/constition/congress)") %>%
  dySeries("topic_15", label = "(public)") %>%
  dySeries("topic_18", label = "(business/economic/tax)") %>%
  dySeries("others", label = "(soviet/german/war)")

 
# perform T-SNE on dataframe of document-topic distritbutions
# the aim is to show a 2-D representation of presidential speeches based on their inferred topics
from sklearn import manifold
from time import time
t0 = time()
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
Y = tsne.fit_transform(df)
t1 = time()
print("t-SNE: %.2g sec" % (t1 - t0))

 
# output T-SNE 2-D representation of each speech to file
president_clusters = pd.DataFrame(Y)
president_clusters = president_clusters.set_index(df.index)
print president_clusters.head(5)
president_clusters.to_csv('speech_tsne_clusters.csv', sep=',')

import matplotlib.pyplot as plt
import numpy as np
import mpld3
import colorsys
from mpld3 import plugins
mpld3.enable_notebook()

presidents = [x.split('|')[0] for x in president_clusters.ix[:,0]]
unique_presidents = list(set(presidents))
N = len(unique_presidents)
HSV_tuples = [(x*1.0/N, 0.7, 0.9) for x in range(N)]
RGB_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), HSV_tuples)

 
president_colors = {}
for i, p in enumerate(unique_presidents):
    president_colors[p] = RGB_tuples[i]

 
fig, ax = plt.subplots(subplot_kw=dict(axisbg='#EEEEEE'), figsize=(10,10))
ax.grid(color='white', linestyle='solid')
scatter = ax.scatter(president_clusters.ix[:, 1],
                  president_clusters.ix[:, 2],
                     s=80,
                     label=presidents,
                     alpha=0.3,
                  c=[president_colors[p] for p in presidents])

ax.grid(color='white', linestyle='solid')
ax.set_title("Clustering presidential speeches", size=20)
labels = ['<h3>{president}</h3>'.format(president=x) for x in presidents] 
tooltip = plugins.PointHTMLTooltip(scatter, labels)
plugins.connect(fig, tooltip)

df = pickle.load( open( "presidential_speeches.pickle", "rb" ) )
speeches = df.transcript.tolist()
presidents = df.speaker.tolist()

from collections import defaultdict
president_words = defaultdict(list)
for i, president in enumerate(presidents):
    president_words[president].append(speeches[i])

president_all_words = []
for key in president_words.keys():
    president_all_words.append(' '.join(president_words[key]).encode('utf8'))

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(min_df=2,
                             ngram_range=(1, 2),
                             strip_accents='unicode',
                             stop_words='english',
                             max_features=5000)
X = vectorizer.fit_transform(president_all_words)
vocab = vectorizer.get_feature_names()
print X.shape

# output T-SNE 2-D representation of each speech to file
president_clusters = pd.DataFrame(Y)
president_clusters['president'] = pd.Series([str(x) for x in president_words.keys()], index=president_clusters.index)
print president_clusters.head(5)
president_clusters.to_csv('president_tsne_clusters.csv', index=False)

# extract unique lits of presidents
presidents = [x for x in president_clusters['president']]
unique_presidents = list(set(presidents))
N = len(unique_presidents)
HSV_tuples = [(x*1.0/N, 0.5, 0.9) for x in range(N)]
RGB_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), HSV_tuples)

# map distinct color to each president
president_colors = {}
for i, p in enumerate(unique_presidents):
    president_colors[p] = RGB_tuples[i]

fig, ax = plt.subplots(subplot_kw=dict(axisbg='#EEEEEE'), figsize=(10,10))
ax.grid(color='white', linestyle='solid')
scatter = ax.scatter(president_clusters.ix[:, 0],
                  president_clusters.ix[:, 1],
                     s=80,
                     label=presidents,
                  c=[president_colors[p] for p in presidents])

ax.grid(color='white', linestyle='solid')
ax.set_title("Clustering presidents based on their speeches", size=20)
labels = ['<h3>{president}</h3>'.format(president=x) for x in presidents] 
tooltip = plugins.PointHTMLTooltip(scatter, labels)
plugins.connect(fig, tooltip)