. . . load the text, and pass to spacy for part-of-speech tagging and lemmatization.
import spacy
nlp = spacy.load('en')
import codecs, re
from collections import defaultdict, Counter
from nltk.corpus import stopwords
sw = set(stopwords.words('english'))
CORPUS_FOLDER = '/home/spenteco/0/corpora/muncie_public_library_corpus/PG_no_backmatter_fiction/'
text = codecs.open(CORPUS_FOLDER + 'Bront_Charlotte_Jane_Eyre_An_Autobiography_PG_1260.txt',
'r', encoding='utf-8').read()
text = re.sub('\s+', ' ', text).strip()
spacy_doc = nlp(text)
all_lemma = []
tag_sequence = []
word_sequence = []
for t in spacy_doc:
if t.text.lower() not in stopwords.words('english') and t.pos_ not in ['PUNCT']:
all_lemma.append(t.lemma_)
tag_sequence.append(t.pos_)
word_sequence.append(t.text)
print 'len(all_lemma)', len(all_lemma)
print 'len(tag_sequence)', len(tag_sequence)
print 'len(word_sequence)', len(word_sequence)
print
print all_lemma[:200]
print
print tag_sequence[:200]
print
print word_sequence[:200]
len(all_lemma) 89267 len(tag_sequence) 231203 len(word_sequence) 231203 [u'jane', u'eyre', u'autobiography', u'charlotte', u'bronte', u'illustrated', u'f.', u'h.', u'townsend', u'london', u'service', u'&', u'paton', u'5', u'henrietta', u'street', u'1897', u'illustrations', u'volume', u'copyright', u'service', u'&', u'paton', u'london', u'w.', u'm.', u'thackeray', u'esq', u'work', u'respectfully', u'inscribed', u'author', u'preface', u'preface', u'first', u'edition', u'jane', u'eyre', u'unnecessary', u'give', u'none', u'second', u'edition', u'demand', u'word', u'acknowledgment', u'miscellaneous', u'remark', u'thank', u'due', u'three', u'quarter', u'public', u'indulgent', u'ear', u'incline', u'plain', u'tale', u'pretension', u'press', u'fair', u'field', u'honest', u'suffrage', u'open', u'obscure', u'aspirant', u'publisher', u'aid', u'tact', u'energy', u'practical', u'sense', u'frank', u'liberality', u'afford', u'unknown', u'unrecommended', u'author', u'press', u'public', u'vague', u'personification', u'must', u'thank', u'vague', u'term', u'publisher', u'definite', u'certain', u'generous', u'critic', u'encourage', u'large', u'hearted', u'high', u'minded', u'man', u'know', u'encourage', u'struggling', u'stranger', u'i.e.', u'publisher', u'select', u'reviewers', u'say', u'cordially', u'gentlemen', u'thank', u'heart', u'thus', u'acknowledge', u'owe', u'aid', u'approve', u'turn', u'another', u'class', u'small', u'one', u'far', u'know', u'therefore', u'overlook', u'mean', u'timorous', u'carping', u'doubt', u'tendency', u'book', u'jane', u'eyre', u'whose', u'eye', u'whatev', u'unusual', u'wrong', u'whose', u'ear', u'detect', u'protest', u'bigotry', u'parent', u'crime', u'insult', u'piety', u'regent', u'god', u'earth', u'would', u'suggest', u'doubter', u'certain', u'obvious', u'distinction', u'would', u'remind', u'certain', u'simple', u'truth', u'conventionality', u'morality', u'self', u'righteousness', u'religion', u'attack', u'first', u'assail', u'last', u'pluck', u'mask', u'face', u'pharisee', u'lift', u'impious', u'hand', u'crown', u'thorns', u'thing', u'deed', u'diametrically', u'oppose', u'distinct', u'vice', u'virtue', u'man', u'often', u'confound', u'confound', u'appearance', u'mistaken', u'truth', u'narrow', u'human', u'doctrine', u'tend', u'elate', u'magnify', u'substitute'] [u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'ADP', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'CCONJ', u'PROPN', u'NUM', u'PROPN', u'PROPN', u'NUM', u'DET', u'PROPN', u'ADP', u'DET', u'NOUN', u'VERB', u'DET', u'NOUN', u'ADP', u'PROPN', u'CCONJ', u'PROPN', u'PUNCT', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PROPN', u'PUNCT', u'PROPN', u'PUNCT', u'PUNCT', u'DET', u'NOUN', u'VERB', u'ADV', u'PROPN', u'ADP', u'DET', u'PROPN', u'PROPN', u'DET', u'NOUN', u'ADP', u'DET', u'ADJ', u'NOUN', u'ADP', u'PUNCT', u'PROPN', u'PROPN', u'PUNCT', u'VERB', u'ADJ', u'PUNCT', u'PRON', u'VERB', u'NOUN', u'PUNCT', u'DET', u'ADJ', u'NOUN', u'VERB', u'DET', u'ADJ', u'NOUN', u'DET', u'ADP', u'NOUN', u'CCONJ', u'ADJ', u'NOUN', u'PUNCT', u'ADJ', u'NOUN', u'VERB', u'ADJ', u'ADP', u'NUM', u'NOUN', u'PUNCT', u'ADP', u'DET', u'PROPN', u'PUNCT', u'ADP', u'DET', u'ADJ', u'NOUN', u'PRON', u'VERB', u'VERB', u'ADP', u'DET', u'ADJ', u'NOUN', u'ADP', u'ADJ', u'NOUN', u'PUNCT', u'ADP', u'DET', u'PROPN', u'PUNCT', u'ADP', u'DET', u'ADJ', u'NOUN', u'ADJ', u'ADJ', u'NOUN', u'VERB', u'VERB', u'ADP', u'DET', u'ADJ', u'NOUN', u'PUNCT', u'ADP', u'ADJ', u'NOUN', u'PUNCT', u'ADP', u'DET', u'NOUN', u'ADJ', u'NOUN', u'PUNCT', u'ADJ', u'NOUN', u'PUNCT', u'ADJ', u'ADJ', u'NOUN', u'CCONJ', u'ADJ', u'NOUN', u'VERB', u'VERB', u'DET', u'NOUN', u'CCONJ', u'ADJ', u'NOUN', u'PUNCT', u'DET', u'PROPN', u'CCONJ', u'DET', u'PROPN', u'VERB', u'CCONJ', u'ADJ', u'NOUN', u'ADP', u'PRON', u'PUNCT', u'CCONJ', u'PRON', u'VERB', u'VERB', u'PRON', u'ADP', u'ADJ', u'NOUN', u'PUNCT', u'CCONJ', u'ADJ', u'NOUN', u'VERB', u'ADJ', u'PUNCT', u'ADV', u'VERB', u'ADJ', u'ADJ', u'NOUN', u'NOUN', u'VERB', u'VERB', u'PRON', u'ADP', u'ADV', u'ADJ', u'PUNCT', u'ADJ', u'CCONJ', u'ADJ', u'PUNCT', u'ADJ'] [u'JANE', u'EYRE', u'AN', u'AUTOBIOGRAPHY', u'BY', u'CHARLOTTE', u'BRONTE', u'ILLUSTRATED', u'BY', u'F.', u'H.', u'TOWNSEND', u'London', u'SERVICE', u'&', u'PATON', u'5', u'HENRIETTA', u'STREET', u'1897', u'The', u'Illustrations', u'in', u'this', u'Volume', u'are', u'the', u'copyright', u'of', u'SERVICE', u'&', u'PATON', u',', u'London', u'TO', u'W.', u'M.', u'THACKERAY', u',', u'ESQ', u'.', u',', u'This', u'Work', u'IS', u'RESPECTFULLY', u'INSCRIBED', u'BY', u'THE', u'AUTHOR', u'PREFACE', u'A', u'preface', u'to', u'the', u'first', u'edition', u'of', u'"', u'Jane', u'Eyre', u'"', u'being', u'unnecessary', u',', u'I', u'gave', u'none', u':', u'this', u'second', u'edition', u'demands', u'a', u'few', u'words', u'both', u'of', u'acknowledgment', u'and', u'miscellaneous', u'remark', u'.', u'My', u'thanks', u'are', u'due', u'in', u'three', u'quarters', u'.', u'To', u'the', u'Public', u',', u'for', u'the', u'indulgent', u'ear', u'it', u'has', u'inclined', u'to', u'a', u'plain', u'tale', u'with', u'few', u'pretensions', u'.', u'To', u'the', u'Press', u',', u'for', u'the', u'fair', u'field', u'its', u'honest', u'suffrage', u'has', u'opened', u'to', u'an', u'obscure', u'aspirant', u'.', u'To', u'my', u'Publishers', u',', u'for', u'the', u'aid', u'their', u'tact', u',', u'their', u'energy', u',', u'their', u'practical', u'sense', u'and', u'frank', u'liberality', u'have', u'afforded', u'an', u'unknown', u'and', u'unrecommended', u'Author', u'.', u'The', u'Press', u'and', u'the', u'Public', u'are', u'but', u'vague', u'personifications', u'for', u'me', u',', u'and', u'I', u'must', u'thank', u'them', u'in', u'vague', u'terms', u';', u'but', u'my', u'Publishers', u'are', u'definite', u':', u'so', u'are', u'certain', u'generous', u'critics', u'who', u'have', u'encouraged', u'me', u'as', u'only', u'large', u'-', u'hearted', u'and', u'high', u'-', u'minded']
import difflib
# PART-OF-SPEECH TAGGING ISN'T HELPING ALL THAT MUCH; NAIVE REGEX FINDS MORE . . .
for a in range(0, len(tag_sequence[:10000])):
if tag_sequence[a] == 'CCONJ':
pre_seq = tag_sequence[a - 3: a]
post_seq = tag_sequence[a + 1: a + 4]
match_length = 0
if pre_seq[-3] == post_seq[0] and pre_seq[-2] == post_seq[1] and pre_seq[-1] == post_seq[0]:
match_length = 3
elif pre_seq[-2] == post_seq[0] and pre_seq[-1] == post_seq[1]:
match_length = 2
elif pre_seq[-1] == post_seq[0] and pre_seq[-1] != 'PUNCT':
match_length = 1
if match_length > 0:
#print
#print 'pre_seq', pre_seq, 'post_seq', post_seq
if match_length == 1:
print ' '.join(word_sequence[a - 1: a] + [word_sequence[a]] + word_sequence[a + 1: a + 2])
if match_length == 2:
print ' '.join(word_sequence[a - 2: a] + [word_sequence[a]] +word_sequence[a + 1: a + 3])
if match_length == 3:
print ' '.join(word_sequence[a - 3: a] + [word_sequence[a]] + word_sequence[a + 1: a + 4])
#print match_length
SERVICE & PATON SERVICE & PATON practical sense and frank liberality The Press and the Public large - hearted and high - minded aided and approved timorous or carping things and deeds elate and magnify broadly and clearly scrutinise and expose Judah and Israel fingers and toes quarrelling nor crying sociable and childlike attractive and sprightly cavillers or questioners mist and cloud lawn and storm long and lamentable rocks and promontories Lindeness , or Naze , frost and snow billow and spray cold and ghastly undeveloped understanding and imperfect feelings love and adventure Pamela , and Henry , vision or conception approach and stand dingy and unwholesome heavy limbs and large extremities dim and bleared bleared eye and flabby cheeks cakes and sweetmeats mother and sisters bullied and punished two or three once or twice his menaces or his inflictions blind and deaf now and then disgusting and ugly suddenly and strongly Go and stand the mirror and the windows lift and poise Eliza and Georgiana my hair and my shoulder Eliza and Georgiana Bessie and Miss darkly and doubtfully painful and crushing Misses Reed and Master Reed useful and pleasant passionate and rude largest and stateliest festoons and falls piled- up mattresses and pillows of the nursery and kitchen the mirrors and the furniture bed and room got up and went to colder and darker face and arms headstrong and selfish captious and insolent pink cheeks and golden curls naughty and tiresome ached and bled eating or drinking dependent and friendless rear and maintain kindly ; and now , punish the perjured and avenge the oppressed Bessie and Abbot Abbot and Bessie submission and stillness Bessie and Abbot frantic anguish and wild sobs wind or water raised or upheld a pillow or an arm protection and security smiling and saying sheltered and befriended come and sleep the fire and the candle severe or prolonged hither and thither now and then ceaseless reprimand and thankless fagging convolvuli and rosebuds long deferred and often wished again and again leaves and bells wilder and thicker Lilliput and Brobdignag houses , and trees , men and women eerie and dreary malevolent and fearful dusting and tidying silk and satin so . But now , far and so distant and soft promise and blessing small and grey eight or nine smile and frown first and only father or mother mother , brothers or sisters . " aunt and cousins
import nltk
from nltk.collocations import *
bigram_measures = nltk.collocations.BigramAssocMeasures()
print dir(bigram_measures)
print
#bigram_finder = BigramCollocationFinder.from_words(all_lemma)
bigram_finder = BigramCollocationFinder.from_words(all_lemma)
bigram_finder.apply_freq_filter(3)
for n in bigram_finder.nbest(bigram_measures.pmi, 10):
print n
print
bigram_finder = BigramCollocationFinder.from_words(all_lemma)
bigram_scored = bigram_finder.score_ngrams(bigram_measures.raw_freq)
scores = []
for b in bigram_scored:
scores.append([b[1], b[0]])
scores.sort(reverse=True)
for s in scores[:25]:
print s
print
bigram_finder = BigramCollocationFinder.from_words(all_lemma)
bigram_scored = bigram_finder.score_ngrams(bigram_measures.likelihood_ratio)
scores = []
for b in bigram_scored:
scores.append([b[1], b[0]])
scores.sort(reverse=True)
for s in scores[:25]:
print s
print
bigram_finder = BigramCollocationFinder.from_words(all_lemma)
bigram_scored = bigram_finder.score_ngrams(bigram_measures.pmi)
scores = []
for b in bigram_scored:
scores.append([b[1], b[0]])
scores.sort(reverse=True)
for s in scores[:25]:
print s
print
bigram_finder = BigramCollocationFinder.from_words(all_lemma)
bigram_scored = bigram_finder.score_ngrams(bigram_measures.mi_like)
scores = []
for b in bigram_scored:
scores.append([b[1], b[0]])
scores.sort(reverse=True)
for s in scores[:25]:
print s
['__abstractmethods__', '__class__', '__delattr__', '__dict__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__module__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_abc_cache', '_abc_negative_cache', '_abc_negative_cache_version', '_abc_registry', '_contingency', '_expected_values', '_marginals', '_n', 'chi_sq', 'dice', 'fisher', 'jaccard', 'likelihood_ratio', 'mi_like', 'phi_sq', 'pmi', 'poisson_stirling', 'raw_freq', 'student_t'] (u'per', u'annum') (u'battle', u'fight') (u"n'est", u'ce') (u'backwards', u'forwards') (u'comme', u'cela') (u'inexpressible', u'sadness') (u"lady's", u'maid') (u'pearl', u'necklace') (u'bitternutt', u'lodge') (u'madame', u'pierrot') [0.0037191795400315906, (u'mr.', u'rochester')] [0.0014563052415786349, (u'st.', u'john')] [0.0013554841094693447, (u'mrs.', u'fairfax')] [0.000907390188983611, (u'mrs.', u'reed')] [0.0007057479247650308, (u'miss', u'temple')] [0.0006497361847043141, (u'miss', u'ingram')] [0.0005489150525950239, (u'rochester', u"'s")] [0.0005041056605464506, (u'mr.', u'brocklehurst')] [0.0005041056605464506, (u'miss', u'eyre')] [0.0004704986165100205, (u'jane', u'eyre')] [0.0003920821804250171, (u'yes', u'sir')] [0.0003696774844007304, (u'thornfield', u'hall')] [0.0003696774844007304, (u'mr.', u'rivers')] [0.00032486809235215703, (u'diana', u'mary')] [0.0003024633963278703, (u'young', u'lady')] [0.0003024633963278703, (u'say', u'mr.')] [0.00028005870030358364, (u'could', u'see')] [0.0002688563522914403, (u'drawing', u'room')] [0.0002576540042792969, (u'mr.', u'mason')] [0.0002576540042792969, (u'miss', u'scatcherd')] [0.0002576540042792969, (u'miss', u'miller')] [0.0002576540042792969, (u'last', u'night')] [0.0002464516562671536, (u'mr.', u'st.')] [0.0002464516562671536, (u'ere', u'long')] [0.00023524930825501026, (u'never', u'see')] [3423.729426826966, (u'mr.', u'rochester')] [1658.9857310778325, (u'st.', u'john')] [1392.2334832651893, (u'mrs.', u'fairfax')] [808.1722016176082, (u'mrs.', u'reed')] [672.5829876109123, (u'miss', u'temple')] [551.0696505647694, (u'miss', u'ingram')] [404.3104105664045, (u'mr.', u'brocklehurst')] [376.544503134581, (u'miss', u'eyre')] [334.7204629254437, (u'&', u'c.')] [333.6897252338787, (u'jane', u'eyre')] [330.12807651947253, (u'thornfield', u'hall')] [305.24517659280593, (u'diana', u'mary')] [283.6938685856298, (u'grace', u'poole')] [268.63788418646743, (u'mr.', u'rivers')] [262.07992788560875, (u'miss', u'scatcherd')] [262.07992788560875, (u'miss', u'miller')] [257.08562272787975, (u'helen', u'burns')] [251.58975188203584, (u'colonel', u'dent')] [228.34314909449256, (u'drawing', u'room')] [228.22495613390242, (u'young', u'lady')] [226.5024942004551, (u'rochester', u"'s")] [214.09962692953462, (u'yes', u'sir')] [201.0794659850292, (u'ere', u'long')] [198.61344307352144, (u'can', u'not')] [192.26004552781683, (u'third', u'storey')] [16.44583932157858, (u'zornes', u'und')] [16.44583932157858, (u'zig', u'zag')] [16.44583932157858, (u'zembla', u'iceland')] [16.44583932157858, (u'yelp', u'whine')] [16.44583932157858, (u'xxi', u'presentiments')] [16.44583932157858, (u'wordy', u'wonderment')] [16.44583932157858, (u'winged', u'condor')] [16.44583932157858, (u'wilt', u'thou')] [16.44583932157858, (u'wield', u'fan')] [16.44583932157858, (u'whimpering', u'moaning')] [16.44583932157858, (u'whatever', u'conte')] [16.44583932157858, (u'werke', u'mit')] [16.44583932157858, (u'welsh', u'rabbit')] [16.44583932157858, (u'wave-', u'girt')] [16.44583932157858, (u'waggon', u'labouring')] [16.44583932157858, (u'w.', u'm.')] [16.44583932157858, (u'virtually', u'nominally')] [16.44583932157858, (u'vinaigrette', u'wield')] [16.44583932157858, (u'vermillion', u'ravished')] [16.44583932157858, (u'veri', u'rascal')] [16.44583932157858, (u'venice', u'vienna')] [16.44583932157858, (u'vehemently', u'clasps')] [16.44583932157858, (u'vais', u'danser')] [16.44583932157858, (u'v.', u'16')] [16.44583932157858, (u'untidy', u'badge')] [183.79524268723884, (u'mr.', u'rochester')] [81.16595241613713, (u'st.', u'john')] [51.31389757849612, (u'mrs.', u'fairfax')] [16.222252747252746, (u'mrs.', u'reed')] [15.041666666666666, (u'&', u'c.')] [11.166800643086816, (u'miss', u'temple')] [7.433756805807622, (u'colonel', u'dent')] [6.970775276884602, (u'miss', u'ingram')] [5.788125, (u'grace', u'poole')] [4.09626216077829, (u'helen', u'burns')] [3.9372759856630823, (u'third', u'storey')] [3.4225714285714286, (u'thornfield', u'hall')] [3.342790570175439, (u'diana', u'mary')] [2.9387577399380804, (u'mr.', u'brocklehurst')] [2.8726120673349724, (u'miss', u'eyre')] [2.56, (u'backwards', u'forwards')] [2.4615384615384617, (u'thousand', u'pound')] [2.25, (u'per', u'annum')] [2.0932361417189354, (u'jane', u'eyre')] [2.0, (u'topsy', u'turvy')] [2.0, (u'spur', u'heel')] [2.0, (u'pre', u'cise')] [2.0, (u'pollard', u'willow')] [2.0, (u'petit', u'coffre')] [2.0, (u'naples', u'florence')]
import re
print
matches = re.finditer(r'\b[a-z]*ing and [a-z]*ing\b', text.lower())
for m in matches:
print text[m.start() - 40: m.end() + 40]
print
matches = re.finditer(r'\b[a-z]*ly and [a-z]*ly\b', text.lower())
for m in matches:
print text[m.start() - 40: m.end() + 40]
print
matches = re.finditer(r'\b[a-z]*ed and [a-z]*ed\b', text.lower())
for m in matches:
print text[m.start() - 40: m.end() + 40]
m at the same time my hand: he took it, smiling and saying, "We shall do very well by-and-by." The untasted tart. Bessie had now finished dusting and tidying the room, and having washed her hands, tion, I contrived to find a pleasure in loving and cherishing a faded graven image, shabby as a minia elivered myself up to the employment of watching and thinking. My reflections were too undefined and uble daisies; the sweetbriars gave out, morning and evening, their scent of spice and apples; and t on of the house below, and succeeded in opening and shutting, without noise, two doors, I reached an nk, Miss Jane?" In another second I was embracing and kissing her rapturously: "Bessie! Bessie! Bessi no notion of sketching a character, or observing and describing salient points, either in persons or th g, clattering process, accompanied by a barking and baying which removed me effectually some yards e of softening the previous outrage, of stroking and soothing me into placidity, you stick a sly penk expects me to talk for the mere sake of talking and showing off, he will find he has addressed hims a kind of innate sympathy; not the less comforting and encouraging because it is very unobtrusive in its m " Here ensued a pause, filled up by the producing and lighting of a cigar; having placed it to his lip much confabulation succeeded a sound of scrubbing and setting to rights; and when I passed the room, y irritated him; I knew the pleasure of vexing and soothing him by turns; it was one I chiefly deli gallery, and soft cheerful laughs, and opening and closing doors, and, for a time, a hush. "Elles ssibly go to sleep while the doors kept opening and shutting below, and people bustling about. Besid bent on striking them as something very dashing and daring indeed. "Oh, I am so sick of the young our eyes -- indeed, they are there now, shining and swimming; and a bead has slipped from the lash a e parties, the less I felt justified in judging and blaming either him or Miss Ingram for acting in it to me. "What are they doing, Jane?" "Laughing and talking, sir." "They don't look grave and myste ne come?" it cried; and then, while the staggering and stamping went on wildly, I distinguished through y longer." And so, by dint of alternate coaxing and commanding, he contrived to get them all once more erent from her sister's -- so much more flowing and becoming -- it looked as stylish as the other's ming heartily like any other child, but whimpering and moaning. Reed pitied it; and he used to nurse i o me? what to Mason? In a state between sleeping and waking, you noticed her entrance and her actio uch steadfast brows, ever revealed such flaming and flashing eyes. I know not whether the day was fa feel the thoughts whose force he seemed breasting and resisting. At the churchyard wicket he stopped: h ed on my cherished wishes, yesterday so blooming and glowing; they lay stark, chill, livid corpses t John, "as they would have a pleasure in keeping and cherishing a half-frozen bird, some wintry wind mi . Then I rose up on my curtainless bed, trembling and quivering; and then the still, dark night witness on't imagine such hard things. Fancy me yielding and melting, as I am doing: human love rising like original materials to the best account; pruning and training nature. But she could not eradicate nat fortably confused; and while I was thus thinking and feeling, St. John bent his head; his Greek face vincing with what skill he could, while acting and speaking apparently just as usual, extract from s, spoken in a cool, tranquil tone, was mortifying and baffling enough. Had I attended to the suggestio deaf, distracted with a revengeful fury tracking and scourging me, on the morning I fled from Thornfie ed, to see with those sightless eyes -- unavailing and distressing attempt! "Answer me -- speak again!" he when she left school, I found in her a pleasing and obliging companion: docile, good-tempered, and w is a good, and not a bad action to mark broadly and clearly the line of separation between them. Th ll at once, without speaking, he struck suddenly and strongly. I tottered, and on regaining my equili s Abbot stood with folded arms, looking darkly and doubtfully on my face, as incredulous of my sanity hall remember how you thrust me back -- roughly and violently thrust me back -- into the red-room, an end; she surveyed the two rows of girls silently and gravely. Miss Miller approaching, seemed to ask unloosed her pinafore, and the teacher instantly and sharply inflicted on her neck a dozen strokes w man who wished to do right could act so unjustly and unwisely as Charles the First sometimes did; and ces. I can remember Miss Temple walking lightly and rapidly along our drooping line, her plaid cloa this charge which Mr. Brocklehurst has weakly and pompously repeated at second-hand from Mrs. Reed; ome died at the school, and were buried quietly and quickly, the nature of the malady forbidding de n my pillow; for as I lay down, it came quietly and naturally to my mind. -- "Those who want situatio , as I entered, the old lady got up and promptly and kindly came forward to meet me. "How do you do allery, when he would sometimes pass me haughtily and coldly, just acknowledging my presence by a di enants one after another: each came out gaily and airily, with dress that gleamed lustrous throu and so much annoyance as she caused me, daily and hourly, with her incomprehensible disposition, ?" I asked. "Because I disliked you too fixedly and thoroughly ever to lend a hand in lifting you to p custom intervened between me and what I naturally and inevitably loved. "It is a long way," I again said a string somewhere under my left ribs, tightly and inextricably knotted to a similar string situated in Mr. Rochester sat quiet, looking at me gently and seriously. Some time passed before he spoke; he a he man obeyed. Mr. Rochester continued, hardily and recklessly: "Bigamy is an ugly word! -- I meant, h n of will. "Yes, sir." "Then tell me so roundly and sharply -- don't spare me." "I cannot: I am tir ed. You shall be Mrs. Rochester -- both virtually and nominally. I shall keep only to you so long as yo do," extricating myself from restraint rapidly and completely. "Oh, Jane, this is bitter! This -- thi tion," and the white door closed, quite gently and civilly: but it shut me out. If she had held it o ladies stopped: Mary, in a few words, kindly and calmly expressed the pleasure she felt in seei John, the intimacy which had arisen so naturally and rapidly between me and his sisters did not exte ed the labours of the village-school as actively and faithfully as I could. It was truly hard work at f lengths." I continued, "You observed it closely and distinctly; but I have no objection to your lookin e found a salutary solace. I feel I can easily and naturally make room in my heart for you, as my th cousins saw at length that my mind was really and immutably fixed on making a just division of the firmed: they manifested their affection plainly and strongly. Deep was my gratification to find I ha strict account. Jane, I shall watch you closely and anxiously -- I warn you of that. And try to restr ot now be explicit about my plans, they kindly and wisely acquiesced in the silence with which I But how they fix! How he starts! How he suddenly and vehemently clasps in both arms the form he dared n He descended the one step, and advanced slowly and gropingly towards the grass-plat. Where was his d acknowledged what I owe those who have aided and approved me, I turn to another class; a small on and sisters, and an antipathy to me. He bullied and punished me; not two or three times in the week, , and from noon to night. My head still ached and bled with the blow and fall I had received: and now, as I sat looking at the white bed and overshadowed walls -- occasionally also turning a fa ay, he departed; to my grief: I felt so sheltered and befriended while he sat in the chair near my pillo to get well: a change seemed near, -- I desired and waited it in silence. It tarried, however: day oved beyond a doubt that I was the most wicked and abandoned child ever reared under a roof. I half f conversation as the drawing-room door opened and closed. When tired of this occupation, I would ve been a meet emblem of my mind when I accused and menaced Mrs. Reed: the same ridge, black and bl were feeding, where the short grass was nipped and blanched. It was a very grey day; a most opaque d for the fourth time: the classes were marshalled and marched into another room to breakfast: how gla he schoolroom and dormitory, was lit by mullioned and latticed windows, which gave it a church-like as atoes and strange shreds of rusty meat, mixed and cooked together. Of this preparation a tolerab melted there: our ungloved hands became numbed and covered with chilblains, as were our feet: I re pelisses and a cloud of silvery plumage extended and waved below me. Mr. Brocklehurst hemmed. "Lad f gall and wormwood than ordinary. Thus restrained and simplified, it sounded more credible: I felt as I s, usually even to my own, I appeared a disciplined and subdued character. But destiny, in the shape of ain to find a response, and quickly. It worked and worked faster: I felt the pulses throb in my h mmittee on my behalf, and which she had framed and glazed. "Well, that is beautiful, Miss Jane! I landed proprietor -- nothing more: she inquired and searched no further, and evidently wondered at m its trampling fore-feet. The traveller waited and watched for some time, and at last he laughed. of a monotonous life. My help had been needed and claimed; I had given it: I was pleased to have me seemed a tribute to my discretion: I regarded and accepted it as such. His deportment had now for the present they hung together somewhat spoiled and tangled. I cannot deny that I grieved for his g n to cry out, "Who is there?" Something gurgled and moaned. Ere long, steps retreated up the galle ake!" I cried. I shook him, but he only murmured and turned: the smoke had stupefied him. Not a mom , held it up, and surveyed the bed, all blackened and scorched, the sheets drenched, the carpet round soon as day dawned. CHAPTER XVI I both wished and feared to see Mr. Rochester on the day which f devour the life that feeds it; and, if discovered and responded to, must lead, ignis-fatus -like, into e charwoman was going on; but here Leah turned and perceived me, and she instantly gave her companio these features appeared to me not only inflated and darkened, but even furrowed with pride; and the ed, and their contents, in the shape of brocaded and hooped petticoats, satin sacques, black modes, e point -- this was where the nerve was touched and teased -- this was where the fever was sustain teased -- this was where the fever was sustained and fed: she could not charm him . If she had m raid." Nor was I; but I was a good deal interested and excited. CHAPTER XIX The library looked tranqui e come?" I cried inwardly, as the night lingered and lingered -- as my bleeding patient drooped, moan in all the daylight he could; and I was surprised and cheered to see how far dawn was advanced: what ain returned, bearing an immense mantle lined and edged with fur. "Now, I've another errand for f all this!" added Mr. Rochester, as he closed and barred the heavy yard-gates. This done, he mov suppose you were no longer a girl well reared and disciplined, but a wild boy indulged from childhood ns in goodness: if any one you know has suffered and erred, let him look higher than his equals fo loined dainty on a nursery chair: and I smiled and obeyed her as in bygone days. She wanted to kn y assured her that I was the person she supposed and desired me to be: and seeing that I was underst rs. Reed in her last moments; I saw her disfigured and discoloured face, and heard her strangely altered v with them?" "Yes." "Pity!" he said, and sighed and paused. "It is always the way of events in thi s. And what ailed the chestnut tree? it writhed and groaned; while wind roared in the laurel walk, s my companions: now I shall revisit it healed and cleansed, with a very angel as my comforter." I only in the evening conferences I thus thwarted and afflicted him. He continued to send for me punctu me. "I think, scathed as you look, and charred and scorched, there must be a little sense of life i peep." So I turned at the door: I saw a robed and veiled figure, so unlike my usual self that it nt we return: all the boxes and luggage arranged and strapped on, and the coachman in his seat." "Yes es, it was Mason himself. Mr. Rochester turned and glared at him. His eye, as I have often said, grovelled, seemingly, on all fours; it snatched and growled like some strange wild animal: but it w an of the name of Rochester. Mr. Mason, astonished and distressed as you may suppose, revealed the real s eyes! How weak my conduct! My eyes were covered and closed: eddying darkness seemed to swim round nd rock you will accordingly become." I cleared and steadied my voice to reply: "All is changed abou e me. "Jane! will you hear reason?" (he stooped and approached his lips to my ear); "because, if you w she had tried me indeed: her character ripened and developed with frightful rapidity; her vices spra , blazed, and the air grew pure. I then framed and fixed a resolution. While I walked under the ressed Mason's wounds that night he was stabbed and worried), are the only two I have ever admitted , her vigilance has been more than once lulled and baffled. The lunatic is both cunning and malign owever, would have succumbed now. I had dared and baffled his fury; I must elude his sorrow: I re inclining a glorious brow earthward. It gazed and gazed on me. It spoke to my spirit: immeasura mered in the yard. The great gates were closed and locked; but a wicket in one of them was only l earing it; Mary's pale brown locks were parted and braided smooth: Diana's duskier tresses covered ecent. My very shoes and stockings were purified and rendered presentable. There were the means of wa r. Mr. St. John, when he saw me, merely bowed and passed through; the two ladies stopped: Mary, of her: I saw the part of instructress pleased and suited her; that of scholar pleased and suited pleased and suited her; that of scholar pleased and suited me no less. Our natures dovetailed: mut greater in this one point than theirs, surprised and charmed them. Mary would sit and watch me by th hen I adhered to principle and law, and scorned and crushed the insane promptings of a frenzied mom t my state of mind changed; the fetters dissolved and dropped from every faculty, leaving nothing of ever her humid gales and vapoury skies generated and screened, justified, in this instance, the term. d regular and delicate lineaments; eyes shaped and coloured as we see them in lovely pictures, larg e, and flicker with resistless emotion. Flushed and kindled thus, he looked nearly as beautiful for at all times accustomed, and which both charmed and benefited them; because, while it elevated them i hoolroom: it was a barn before I had it repaired and altered, and barns are generally haunted by rat clock: he did not return till midnight. Starved and tired enough he was: but he looked happier th But as it is, either our union must be consecrated and sealed by marriage, or it cannot exist: practi and a trembling trouble of grief, which harassed and crushed me altogether. I felt how -- if I were iently convince me how completely I was banished and banned, he added the force of contrast; and th en but torpor, from which they were now summoned and forced to wake. They rose expectant: eye and e salubrious site. Ferndean then remained uninhabited and unfurnished, with the exception of some two or thre nd brooding -- that reminded me of some wronged and fettered wild beast or bird, dangerous to approa ommotion was: but as he saw nothing, he returned and sighed. "Give me the water, Mary," he said. I , my shoulder -- neck -- waist -- I was entwined and gathered to him. "Is it Jane? What is it? This i dawned on his forehead: his lineaments softened and warmed. After supper, he began to ask me many er entertained in your life: quite more refined and exalted." "Who the deuce have you been with?" " ts little casket; your trunks were left corded and locked as they had been prepared for the brida le, have you, reader? I had not; I soon asked and obtained leave of Mr. Rochester, to go and see h
import nltk
tokens = nltk.word_tokenize(text.lower())
text_obj = nltk.Text(tokens)
print
text_obj.concordance('and', lines=40, width=115)
print
text_obj.concordance('or', lines=40, width=115)
#print
#text_obj.concordance('but', lines=40, width=115)
Displaying 40 of 6623 matches: cond edition demands a few words both of acknowledgment and miscellaneous remark . my thanks are due in three quart e aid their tact , their energy , their practical sense and frank liberality have afforded an unknown and unrecomme cal sense and frank liberality have afforded an unknown and unrecommended author . the press and the public are but fforded an unknown and unrecommended author . the press and the public are but vague personifications for me , and and the public are but vague personifications for me , and i must thank them in vague terms ; but my publishers ar us critics who have encouraged me as only large-hearted and high-minded men know how to encourage a struggling stra truggling stranger ; to them , i.e . , to my publishers and the select reviewers , i say cordially , gentlemen , i aving thus acknowledged what i owe those who have aided and approved me , i turn to another class ; a small one , s t an impious hand to the crown of thorns . these things and deeds are diametrically opposed : they are as distinct ruth ; narrow human doctrines , that only tend to elate and magnify a few , should not be substituted for the world d of christ . there is -- i repeat it -- a difference ; and it is a good , and not a bad action to mark broadly and is -- i repeat it -- a difference ; and it is a good , and not a bad action to mark broadly and clearly the line o and it is a good , and not a bad action to mark broadly and clearly the line of separation between them . the world clean shrines . it may hate him who dares to scrutinise and expose -- to rase the gilding , and show base metal und dares to scrutinise and expose -- to rase the gilding , and show base metal under it -- to penetrate the sepulchre how base metal under it -- to penetrate the sepulchre , and reveal charnel relics : but hate as it will , it is ind loody death , had he but stopped his ears to flattery , and opened them to faithful counsel . there is a man in our the son of imlah came before the throned kings of judah and israel ; and who speaks truth as deep , with a power as lah came before the throned kings of judah and israel ; and who speaks truth as deep , with a power as prophet-like who speaks truth as deep , with a power as prophet-like and as vital -- a mien as dauntless and as daring . is the wer as prophet-like and as vital -- a mien as dauntless and as daring . is the satirist of `` vanity fair '' admire e amongst whom he hurls the greek fire of his sarcasm , and over whom he flashes the levin-brand of his denunciatio , because i think i see in him an intellect profounder and more unique than his contemporaries have yet recognised to me , an honour is awarded where it is not merited ; and consequently , denied where it is justly due . this exp to rectify mistakes which may already have been made , and to prevent future errors . currer bell . april 13 th , cold winter wind had brought with it clouds so sombre , and a rain so penetrating , that further out-door exercise e coming home in the raw twilight , with nipped fingers and toes , and a heart saddened by the chidings of bessie , me in the raw twilight , with nipped fingers and toes , and a heart saddened by the chidings of bessie , the nurse heart saddened by the chidings of bessie , the nurse , and humbled by the consciousness of my physical inferiority sciousness of my physical inferiority to eliza , john , and georgiana reed . the said eliza , john , and georgiana a , john , and georgiana reed . the said eliza , john , and georgiana were now clustered round their mama in the dr ing-room : she lay reclined on a sofa by the fireside , and with her darlings about her ( for the time neither quar at a distance ; but that until she heard from bessie , and could discover by her own observation , that i was ende endeavouring in good earnest to acquire a more sociable and childlike disposition , a more attractive and sprightly sociable and childlike disposition , a more attractive and sprightly manner -- something lighter , franker , more ng up her elders in that manner . be seated somewhere ; and until you can speak pleasantly , remain silent . '' a b thering up my feet , i sat cross-legged , like a turk ; and , having drawn the red moreen curtain nearly close , i nter afternoon . afar , it offered a pale blank of mist and cloud ; near a scene of wet lawn and storm-beat shrub , pale blank of mist and cloud ; near a scene of wet lawn and storm-beat shrub , with ceaseless rain sweeping away wi with ceaseless rain sweeping away wildly before a long and lamentable blast . i returned to my book -- bewick 's h Displaying 40 of 637 matches: ot , therefore , to be overlooked . i mean the timorous or carping few who doubt the tendency of such books as `` enunciation , were to take his warnings in time -- they or their seed might yet escape a fatal rimoth-gilead . why e done ? '' i asked . `` jane , i do n't like cavillers or questioners ; besides , there is something truly forbid ith isles from its southern extremity , the lindeness , or naze , to the north cape -- `` where the northern ocean dventure taken from old fairy tales and other ballads ; or ( as at a later period i discovered ) from the pages of ound it out himself ; he was not quick either of vision or conception ; but eliza just put her head in at the door at school ; but his mama had taken him home for a month or two , `` on account of his delicate health . '' mr. mil antipathy to me . he bullied and punished me ; not two or three times in the week , nor once or twice in the day shed me ; not two or three times in the week , nor once or twice in the day , but continually : every nerve i had use i had no appeal whatever against either his menaces or his inflictions ; the servants did not like to offend t lind and deaf on the subject : she never saw him strike or heard him abuse me , though he did both now and then in d , and on regaining my equilibrium retired back a step or two from his chair . `` that is for your impudence in a ves : for they are mine ; all the house belongs to me , or will do in a few years . go and stand by the door , out really saw in him a tyrant , a murderer . i felt a drop or two of blood from my head trickle down my neck , and wa in of me . the fact is , i was a trifle beside myself ; or rather out of myself , as the french would say : i was cape from insupportable oppression -- as running away , or , if that could not be effected , never eating or drink way , or , if that could not be effected , never eating or drinking more , and letting myself die . what a conster nobody there ; i had nothing in harmony with mrs. reed or her children , or her chosen vassalage . if they did no had nothing in harmony with mrs. reed or her children , or her chosen vassalage . if they did not love me , in fac a useless thing , incapable of serving their interest , or adding to their pleasure ; a noxious thing , cherishing h ? that certainly was a crime : and was i fit to die ? or was the vault under the chancel of gateshead church an d , might quit its abode -- whether in the church vault or in the unknown world of the departed -- and rise before grief might waken a preternatural voice to comfort me , or elicit from the gloom some haloed face , bending over m th a hollow sound , and as if muffled by a rush of wind or water : agitation , uncertainty , and an all-predominat re , and that more tenderly than i had ever been raised or upheld before . i rested my head against a pillow or an ed or upheld before . i rested my head against a pillow or an arm , and felt easy . in five minutes more the cloud rough . `` i will try . '' `` would you like to drink , or could you eat anything ? '' `` no , thank you , bessie read : such dread as children only can feel . no severe or prolonged bodily illness followed this incident of the d . ev'n should i fall o'er the broken bridge passing , or stray in the marshes , by false lights beguiled , still a n't she manage to walk at her age ? she must be eight or nine years old . '' `` i was knocked down , '' was the t , true response . `` for one thing , i have no father or mother , brothers or sisters . '' `` you have a kind au ` for one thing , i have no father or mother , brothers or sisters . '' `` you have a kind aunt and cousins . '' a f the poor women i saw sometimes nursing their children or washing their clothes at the cottage doors of the villa nuckles could inflict ; and when i saw that either that or my look daunted him , i had the greatest inclination to not worthy of notice ; i do not choose that either you or your sisters should associate with her . '' here , lean dared me in an emphatic voice to rise from that place , or utter one syllable during the remainder of the day . `` t me as if she really did not know whether i were child or fiend . i was now in for it . `` my uncle reed is in he and afterwards , in listening to the sound of the piano or the harp played below , to the passing to and fro of th s she would come up in the interval to seek her thimble or her scissors , or perhaps to bring me something by way p in the interval to seek her thimble or her scissors , or perhaps to bring me something by way of supper -- a bun
from collections import defaultdict, Counter
letter_ngrams = defaultdict(int)
for a in range(0, len(text) - 6 + 1):
letter_ngrams[text[a: a + 6].lower()] += 1
for w in Counter(letter_ngrams).most_common(500):
print w[0], w[1]
, and 2387 that 1513 with 1381 n the 1133 have 1055 of th 1048 of the 901 d the 898 and t 875 the s 871 ed to 866 f the 843 in th 800 from 715 in the 697 and s 685 and th 683 your 665 t the 665 would 661 would 645 ; and 638 the c 632 e was 615 were 597 this 595 nd the 590 to the 587 which 586 to th 586 ed the 582 what 582 which 559 will 557 and i 554 e and 544 when 539 o the 539 t was 522 there 522 and a 516 d not 514 could 504 i had 502 the p 499 i had 498 e, and 492 could 485 they 484 the f 483 and w 481 the w 468 the d 466 i was 465 i was 460 r the 456 though 449 e the 439 thoug 439 s and 427 the l 423 the m 422 it was 419 the t 419 it wa 414 the h 411 there 408 the r 406 said 400 on th 400 at the 391 and h 385 the b 385 s the 382 d and 381 been 380 on the 378 was a 377 to be 376 cheste 372 you a 372 ochest 371 thing 371 hester 371 roches 371 roche 370 ing th 369 ; but 368 the g 361 some 360 e you 358 s, and 356 very 352 their 351 mr. r 349 ought 349 their 347 . roch 342 e had 342 little 342 at th 342 littl 337 like 337 mr. ro 334 , the 333 r. roc 332 d with 330 and i 328 should 326 shoul 324 ould n 323 ittle 322 d, and 321 i have 318 again 317 that i 317 he was 316 and m 314 . the 313 more 313 ng the 313 then 313 hought 313 to be 308 ed in 307 must 299 never 298 , but 298 it is 291 was t 290 e of t 289 h the 288 never 288 befor 287 before 287 and c 286 than 285 uld no 285 d her 284 ing to 284 i hav 283 ation 281 hould 281 only 280 s not 279 and f 278 to me 276 for t 274 ld not 273 ng to 265 the e 265 you w 264 miss 263 other 263 of a 262 into 261 was s 260 i coul 258 t and 258 e with 257 d that 257 he had 256 and b 256 still 255 for th 255 me to 255 the a 255 g the 253 in a 253 i cou 253 ou are 253 you ar 253 after 253 she w 252 d you 251 from t 250 think 249 was n 248 i am 247 efore 246 but i 245 or the 244 such 243 with t 242 u are 242 hat i 242 mrs. 241 er the 240 rom th 239 ed me 238 where 238 shall 237 as the 237 as i 237 shall 236 it is 234 over 232 with a 231 heart 230 the n 230 had b 228 ith th 226 the co 225 you h 225 ed her 223 ed wit 223 looked 223 the o 223 and wh 222 where 221 was no 221 them 221 you ha 220 looke 219 m the 219 om the 219 t of t 218 and p 218 you t 218 about 217 n and 215 ooked 214 r and 214 t you 213 under 212 hen i 211 eemed 210 seemed 210 on of 210 seeme 210 and l 209 and d 209 house 209 , jane 205 ed and 204 ed my 204 somet 204 , with 201 s were 201 y the 201 d his 200 d to t 198 ed by 198 ould b 198 , you 196 " said 196 you s 196 "you 195 here w 195 about 195 ce of 193 -- th 193 ion of 191 other 190 had be 190 th the 190 you, 188 first 188 mysel 187 t, and 187 , that 187 pleas 186 still 186 here i 186 myself 186 with h 184 long 184 ; the 183 was a 183 ould h 183 y and 182 when i 182 ester 182 uld be 181 but i 181 he ha 181 heard 180 think 180 night 180 answer 180 n, and 179 answe 179 othing 179 know 178 e said 178 after 178 re the 177 in my 177 n his 177 me to 176 the ho 176 might 175 much 175 e woul 174 by th 174 was th 174 she h 174 of my 174 down 174 r, and 173 the mo 173 u have 173 ut the 173 ou hav 173 ed, an 172 strang 172 she s 171 "and 171 presen 171 the i 171 in my 170 all th 170 the re 170 but t 170 for a 169 to re 169 ering 169 he wa 169 and r 169 n her 169 trange 168 again 168 she ha 168 all t 168 and a 167 heard 166 the fi 166 nothin 166 er to 166 turned 165 to my 165 stran 165 d for 165 of my 165 while 165 tion, 165 to he 164 well 164 ed at 164 i sho 164 re was 163 d have 163 i shou 162 e that 162 prese 162 ld be 162 time 162 ," sai 161 tion o 160 ather 160 he is 160 to my 160 that t 160 uld ha 160 with m 159 good 159 she wa 159 d then 159 and g 159 o you 159 ing in 158 the la 158 these 158 came 158 had n 158 come 158 d been 157 not t 157 cause 157 e for 157 of he 157 nothi 156 these 156 ecause 156 : the 156 es of 156 he sai 156 not a 156 for m 156 , whic 156 becaus 156 asked 155 while 155 the v 155 ith a 155 ad bee 154 and n 154 ll the 153 hat th 153 you m 153 as not 152 to hi 152 the wo 152 child 151 might 151 i will 151 light 151 don't 151 by the 151 nd you 151 the st 150 just 150 and he 150 felt 150 is no 150 made 149 id not 149 re you 148 ere wa 148 ertain 148 ld hav 148 , when 148 the ro 148 n you 148 under 147 , she 147 what i 147 to se 146 e in t 146 return 145 nt of 145 inter 145 make 144 first 144 en the 144 did no 143 the pa 143 retur 143 throu 143 d to m 143 hrough 143 throug 143 thing 143 her a 142 . john 142 he con 142 ection 142 : but 142 did n 141 withou 141 i can 141 ithout 141 and o 141 rough 141 y, and 141 moment 140 but th 140 er and 140 momen 140 witho 140 not s 140 take 140 ted to 139 that s 139 he fir 139 -- i 139 don't 139 s of t 139 ll be 139 of hi 139 becau 138 e; and 138 once 138 er of 138 ished 138 place 138 every 138 ng of 138 quite 138 urned 137 fairfa 137 airfax 137 have a 137 had a 137 nd to 137 omethi 136 , for 136 last 136 thout 136 fairf 136 ething 136 methin 136 ing of 136 quite 136 someth 136 hich i 135 cannot 135 canno 135 had t 135 ve you 135 l the 134 being 134 and so 134 t her 134 t she 134 n that 134 resent 134 left 134 f you 133 . she 133