12_arc_diagram_explain_results¶
Load spacy, . . .¶
. . . load the text, and pass to spacy for part-of-speech tagging and lemmatization.
In [4]:
import spacy
nlp = spacy.load('en')
In [5]:
import codecs, re
from collections import defaultdict, Counter
from nltk.corpus import stopwords
def get_tokens_for_analysis(FILE_NAME, WORD_COUNT_THRESHOLD):
sw = set(stopwords.words('english'))
CORPUS_FOLDER = '/home/spenteco/0/corpora/muncie_public_library_corpus/PG_no_backmatter_fiction/'
text = codecs.open(CORPUS_FOLDER + FILE_NAME,
'r', encoding='utf-8').read()
text = re.sub('\s+', ' ', text).strip()
spacy_doc = nlp(text)
n_tokens = 0
word_counts = defaultdict(int)
tokens = []
raw_tokens = []
for t in spacy_doc:
n_tokens += 1
raw_tokens.append(t.text)
if t.text.lower() not in stopwords.words('english') and t.pos_ not in ['PUNCT']:
if t.lemma_ == '-PRON-':
word_counts[t.text.lower()] += 1
tokens.append(t.text.lower())
else:
word_counts[t.lemma_.lower()] += 1
tokens.append(t.lemma_.lower())
else:
tokens.append(' ')
slice_size = (n_tokens / 200) + 1
print
print FILE_NAME, 'WORD_COUNT_THRESHOLD', WORD_COUNT_THRESHOLD
print 'slice_size', slice_size
print 'len(tokens)', len(tokens)
print 'len(word_counts)', len(word_counts)
tokens_for_analysis = []
for k, v in word_counts.iteritems():
if v >= WORD_COUNT_THRESHOLD:
tokens_for_analysis.append(k)
print 'len(tokens_for_analysis)', len(tokens_for_analysis)
#print
#print tokens_for_analysis
tokens_for_analysis = set(tokens_for_analysis)
return tokens_for_analysis, tokens, raw_tokens, slice_size
In [6]:
def get_documents(tokens, raw_tokens, slice_size):
documents = []
for a in range(0, 200):
from_a = a * slice_size
to_a = from_a + slice_size
doc = []
for t in tokens[from_a: to_a]:
if t in tokens_for_analysis:
doc.append(t)
documents.append(doc)
display_documents = []
for a in range(0, 200):
doc = []
from_a = a * slice_size
to_a = from_a + slice_size
doc = []
for t in raw_tokens[from_a: to_a]:
if t in tokens_for_analysis:
doc.append('<b>' + t + '</b>')
else:
doc.append(t)
display_documents.append(' '.join(doc))
return documents, display_documents
In [7]:
from gensim import corpora, models, similarities
def get_gensim_objects(documents):
dictionary = corpora.Dictionary(documents)
corpus = [dictionary.doc2bow(doc) for doc in documents]
index = similarities.MatrixSimilarity(corpus)
return dictionary, corpus, index
In [8]:
%matplotlib inline
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
import unicodecsv as csv
import seaborn as sns
from pylab import rcParams
def plot_similarities(corpus, index):
sns.set(color_codes=True)
rcParams['figure.figsize'] = 10, 5
distances = []
distances_amounts_only = []
for doc_n, doc in enumerate(corpus):
sims = index[doc]
for s in list(enumerate(sims)):
if doc_n != s[0]:
distances.append([doc_n, s[0], s[1]])
distances_amounts_only.append(s[1])
np_avg = np.average(distances_amounts_only)
np_std = np.std(distances_amounts_only)
print np_avg
print np.median(distances_amounts_only)
print np.std(distances_amounts_only)
print '1 std', (np_avg + np_std)
print '2 std', (np_avg + (np_std * 2))
distances_for_arc_diagram = []
n_1 = 0
n_2 = 0
n_3 = 0
n_selected = 0
for dn, d in enumerate(distances_amounts_only):
if d >= (np_avg + np_std):
n_1 += 1
if d >= (np_avg + (np_std * 2)):
n_2 += 1
if d >= (np_avg + (np_std * 3)):
n_3 += 1
if d >= (np_avg + (np_std * 2)):
n_selected += 1
distances_for_arc_diagram.append(distances[dn])
print 'n_1', n_1
print 'n_2', n_2
print 'n_3', n_3
print 'n_selected', n_selected
sns.distplot(distances_amounts_only)
plt.show()
return distances_for_arc_diagram
In [9]:
import json
def write_js_objects(FILE_NAME, distances_for_arc_diagram):
nodes = []
for d in range(0, 200):
nodes.append(d)
node_index = {}
for a, n in enumerate(nodes):
node_index[n] = a
nodes = sorted(list(set(nodes)))
network_output = {'directed': False, 'graph': {}, 'nodes': [], 'links': []}
for n in sorted(nodes):
network_output['nodes'].append({'name': str(n), 'group': 1})
for e in distances_for_arc_diagram:
network_output['links'].append({'source': str(e[0]),
'target': str(e[1]),
'value': 1})
f = codecs.open('../tatlock_spring_2018_results/mega_arc_diagrams/' + \
FILE_NAME.replace('.txt', '') + '.js', 'w', encoding='utf-8')
f.write(json.dumps(network_output, indent=4))
f.close()
f = codecs.open('../tatlock_spring_2018_results/mega_arc_diagrams/' + \
FILE_NAME.replace('.txt', '') + '_display_documents.js', 'w', encoding='utf-8')
f.write('var display_documents = ' + json.dumps(display_documents, indent=4))
f.close()
In [10]:
file_names = [
'Bront_Charlotte_Jane_Eyre_An_Autobiography_PG_1260.txt',
'Bront_Charlotte_Shirley_PG_30486.txt',
'Bront_Charlotte_Villette_PG_9182.txt',
'Marlitt_Wister_At_the_Councillor_s_or_A_Nameless_History_PG_43393_0.txt',
'Marlitt_Wister_Baliff.txt',
'Marlitt_Wister_Countess_Gisela_corrected_4_10_2018.txt',
'Marlitt_Wister_Gold_Elsie_PG_42426.txt',
'Marlitt_Wister_Im Schillingshof_4_26_2018.txt',
'Marlitt_Wister_Lady_with_the_Rubies_corrected_3_13_208.txt',
'Marlitt_Wister_Little_Moorland_Princess_cleaned_121817.txt',
'Marlitt_Wister_OMS_translation_cleaned_110617.txt',
'Marlitt_Wister_Owls_Nest_corrected_4_21_2018.txt',
'Marlitt_Wister_The_Second_Wife_corrected.txt',
'Dickens_Charles_David_Copperfield_PG_766.txt',
'Thackeray_William_Makepeace_Vanity_Fair_PG_599.txt',
]
for file_name in file_names:
tokens_for_analysis, tokens, raw_tokens, slice_size = get_tokens_for_analysis(file_name, 0)
documents, display_documents = get_documents(tokens, raw_tokens, slice_size)
dictionary, corpus, index = get_gensim_objects(documents)
distances_for_arc_diagram = plot_similarities(corpus, index)
write_js_objects(file_name, distances_for_arc_diagram)
In [ ]: