TextPreprocessing

ipynb46 kBPython · 47 cellsExtra material of chapter 4

In [35]
from nltk import WordNetLemmatizer
lemma = WordNetLemmatizer()

words = ["study", "studying", "studied"]
for word in words:
    print(lemma.lemmatize(word, pos='v'))
study
study
study
In [30]
import spacy
nlp = spacy.load('en_core_web_sm')
words = "Studies Studying Studied"
lemmas=[token.lemma_ for token in nlp(words)]
print(lemmas)
['study', 'study', 'study']
In [3]
import spacy
 
#load the small English model
nlp = spacy.load("en_core_web_sm")

sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""

pos = [(token.text, token.pos_) for token in nlp(sentence)]
print(pos)
[('Once', 'ADV'), ('upon', 'SCONJ'), ('a', 'DET'), ('time', 'NOUN'), ('there', 'PRON'), ('was', 'VERB'), ('an', 'DET'), ('old', 'ADJ'), ('mother', 'NOUN'), ('pig', 'NOUN'), ('who', 'PRON'), ('had', 'VERB'), ('three', 'NUM'), ('little', 'ADJ'), ('pigs', 'NOUN'), ('and', 'CCONJ'), ('not', 'PART'), ('enough', 'ADJ'), ('food', 'NOUN'), ('to', 'PART'), ('feed', 'VERB'), ('them', 'PRON'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('So', 'ADV'), ('when', 'SCONJ'), ('they', 'PRON'), ('were', 'AUX'), ('old', 'ADJ'), ('enough', 'ADV'), (',', 'PUNCT'), ('she', 'PRON'), ('sent', 'VERB'), ('them', 'PRON'), ('out', 'ADP'), ('into', 'ADP'), ('the', 'DET'), ('world', 'NOUN'), ('to', 'PART'), ('seek', 'VERB'), ('their', 'PRON'), ('fortunes', 'NOUN'), ('.', 'PUNCT')]
In [35]
import nltk
from nltk.tokenize import word_tokenize
content = "Machine learning is a field of study in artificial intelligence."
tokenized_text = nltk.word_tokenize(content)
tagged_token = nltk.pos_tag(tokenized_text)
grammer = r"""
            NP: {<DT>?<JJ>+<NN>}
            {<DT>+<NN>}
            {<NN>+}
            """
phrases = nltk.RegexpParser(grammer)
result = phrases.parse(tagged_token)
print(result)
result.draw()
(S
  (NP Machine/NN learning/NN)
  is/VBZ
  (NP a/DT field/NN)
  of/IN
  (NP study/NN)
  in/IN
  (NP artificial/JJ intelligence/NN)
  ./.)
In [40]
import spacy
from spacy import displacy

nlp = spacy.load("en_core_web_sm")
doc = nlp("Bob ate apple.")
# Since this is an interactive Jupyter environment, we can use displacy.render here
displacy.render(doc, style='dep')
<IPython.core.display.HTML object>

NLTK sentence segmentation

In [2]
from nltk.tokenize import sent_tokenize
text = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
tokenized_text=sent_tokenize(text)
print(tokenized_text[0])
Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them.

spacy sentence segmentation

In [3]
import spacy
nlp = spacy.load('en_core_web_sm') # or whatever model you have installed

text = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""

doc = nlp(text)
sentences = [sent.text.strip() for sent in doc.sents]
print(sentences[0])
Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them.

NLTK word tokenization

In [6]
from nltk.tokenize import word_tokenize
text = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
tokenized_text=word_tokenize(text)
print(tokenized_text)
['Once', 'upon', 'a', 'time', 'there', 'was', 'an', 'old', 'mother', 'pig', 'who', 'had', 'three', 'little', 'pigs', 'and', 'not', 'enough', 'food', 'to', 'feed', 'them', '.', 'So', 'when', 'they', 'were', 'old', 'enough', ',', 'she', 'sent', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortunes', '.', 'The', 'first', 'little', 'pig', 'was', 'very', 'lazy', '.', 'He', 'did', "n't", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'straw', '.', 'The', 'second', 'little', 'pig', 'worked', 'a', 'little', 'bit', 'harder', 'but', 'he', 'was', 'somewhat', 'lazy', 'too', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'sticks', '.', 'Then', ',', 'they', 'sang', 'and', 'danced', 'and', 'played', 'together', 'the', 'rest', 'of', 'the', 'day', '.', 'The', 'third', 'little', 'pig', 'worked', 'hard', 'all', 'day', 'and', 'built', 'his', 'house', 'with', 'bricks', '.', 'It', 'was', 'a', 'sturdy', 'house', 'complete', 'with', 'a', 'fine', 'fireplace', 'and', 'chimney', '.', 'It', 'looked', 'like', 'it', 'could', 'withstand', 'the', 'strongest', 'winds', '.']

spacy word tokenization

In [5]
import spacy
 
#load the small English model
nlp = spacy.load("en_core_web_sm")

sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
doc = nlp(sentence)
tokens = [token.text for token in doc]
print(tokens)
['Once', 'upon', 'a', 'time', 'there', 'was', 'an', 'old', 'mother', 'pig', 'who', 'had', 'three', 'little', 'pigs', 'and', 'not', 'enough', 'food', 'to', 'feed', 'them', '.', '\n', 'So', 'when', 'they', 'were', 'old', 'enough', ',', 'she', 'sent', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortunes', '.', '\n', 'The', 'first', 'little', 'pig', 'was', 'very', 'lazy', '.', 'He', 'did', "n't", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'straw', '.', '\n', 'The', 'second', 'little', 'pig', 'worked', 'a', 'little', 'bit', 'harder', 'but', 'he', 'was', 'somewhat', 'lazy', 'too', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'sticks', '.', '\n', 'Then', ',', 'they', 'sang', 'and', 'danced', 'and', 'played', 'together', 'the', 'rest', 'of', 'the', 'day', '.', '\n', 'The', 'third', 'little', 'pig', 'worked', 'hard', 'all', 'day', 'and', 'built', 'his', 'house', 'with', 'bricks', '.', '\n', 'It', 'was', 'a', 'sturdy', 'house', 'complete', 'with', 'a', 'fine', 'fireplace', 'and', 'chimney', '.', '\n', 'It', 'looked', 'like', 'it', 'could', 'withstand', 'the', 'strongest', 'winds', '.']

NLTK stop words removal

In [7]
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
 
stop_words = set(stopwords.words('english'))
# print(stop_words)
 
word_tokens = word_tokenize(sentence)
filtered_tokens = [word for word in word_tokens if not word.lower() in stop_words]
print(filtered_tokens)
['upon', 'time', 'old', 'mother', 'pig', 'three', 'little', 'pigs', 'enough', 'food', 'feed', '.', 'old', 'enough', ',', 'sent', 'world', 'seek', 'fortunes', '.', 'first', 'little', 'pig', 'lazy', '.', "n't", 'want', 'work', 'built', 'house', 'straw', '.', 'second', 'little', 'pig', 'worked', 'little', 'bit', 'harder', 'somewhat', 'lazy', 'built', 'house', 'sticks', '.', ',', 'sang', 'danced', 'played', 'together', 'rest', 'day', '.', 'third', 'little', 'pig', 'worked', 'hard', 'day', 'built', 'house', 'bricks', '.', 'sturdy', 'house', 'complete', 'fine', 'fireplace', 'chimney', '.', 'looked', 'like', 'could', 'withstand', 'strongest', 'winds', '.']

spacy stop words removal

In [6]
import spacy
 
#load the small English model
nlp = spacy.load("en_core_web_sm")

sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""

doc = nlp(sentence)
filtered_tokens = [token.text for token in doc if not token.is_stop]
print(filtered_tokens)
['time', 'old', 'mother', 'pig', 'little', 'pigs', 'food', 'feed', '.', '\n', 'old', ',', 'sent', 'world', 'seek', 'fortunes', '.', '\n', 'little', 'pig', 'lazy', '.', 'want', 'work', 'built', 'house', 'straw', '.', '\n', 'second', 'little', 'pig', 'worked', 'little', 'bit', 'harder', 'somewhat', 'lazy', 'built', 'house', 'sticks', '.', '\n', ',', 'sang', 'danced', 'played', 'rest', 'day', '.', '\n', 'little', 'pig', 'worked', 'hard', 'day', 'built', 'house', 'bricks', '.', '\n', 'sturdy', 'house', 'complete', 'fine', 'fireplace', 'chimney', '.', '\n', 'looked', 'like', 'withstand', 'strongest', 'winds', '.']

NLTK Stemming

In [12]
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
ps = PorterStemmer()
word_tokens = word_tokenize(sentence)
stem_tokens = [ps.stem(word) for word in word_tokens]
print(stem_tokens)
['onc', 'upon', 'a', 'time', 'there', 'wa', 'an', 'old', 'mother', 'pig', 'who', 'had', 'three', 'littl', 'pig', 'and', 'not', 'enough', 'food', 'to', 'feed', 'them', '.', 'So', 'when', 'they', 'were', 'old', 'enough', ',', 'she', 'sent', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortun', '.', 'the', 'first', 'littl', 'pig', 'wa', 'veri', 'lazi', '.', 'He', 'did', "n't", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'built', 'hi', 'hous', 'out', 'of', 'straw', '.', 'the', 'second', 'littl', 'pig', 'work', 'a', 'littl', 'bit', 'harder', 'but', 'he', 'wa', 'somewhat', 'lazi', 'too', 'and', 'he', 'built', 'hi', 'hous', 'out', 'of', 'stick', '.', 'then', ',', 'they', 'sang', 'and', 'danc', 'and', 'play', 'togeth', 'the', 'rest', 'of', 'the', 'day', '.', 'the', 'third', 'littl', 'pig', 'work', 'hard', 'all', 'day', 'and', 'built', 'hi', 'hous', 'with', 'brick', '.', 'It', 'wa', 'a', 'sturdi', 'hous', 'complet', 'with', 'a', 'fine', 'fireplac', 'and', 'chimney', '.', 'It', 'look', 'like', 'it', 'could', 'withstand', 'the', 'strongest', 'wind', '.']

spacy Stemming

In [15]
# no stemming in spacy

NLTK POS tagging

In [9]
from nltk.tokenize import word_tokenize
from nltk import pos_tag
sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
word_tokens = word_tokenize(sentence)
pos_tagging = pos_tag(word_tokens)
print(pos_tagging)
[('Once', 'RB'), ('upon', 'IN'), ('a', 'DT'), ('time', 'NN'), ('there', 'EX'), ('was', 'VBD'), ('an', 'DT'), ('old', 'JJ'), ('mother', 'NN'), ('pig', 'NN'), ('who', 'WP'), ('had', 'VBD'), ('three', 'CD'), ('little', 'JJ'), ('pigs', 'NNS'), ('and', 'CC'), ('not', 'RB'), ('enough', 'RB'), ('food', 'NN'), ('to', 'TO'), ('feed', 'VB'), ('them', 'PRP'), ('.', '.'), ('So', 'RB'), ('when', 'WRB'), ('they', 'PRP'), ('were', 'VBD'), ('old', 'JJ'), ('enough', 'RB'), (',', ','), ('she', 'PRP'), ('sent', 'VBD'), ('them', 'PRP'), ('out', 'RP'), ('into', 'IN'), ('the', 'DT'), ('world', 'NN'), ('to', 'TO'), ('seek', 'VB'), ('their', 'PRP$'), ('fortunes', 'NNS'), ('.', '.'), ('The', 'DT'), ('first', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('was', 'VBD'), ('very', 'RB'), ('lazy', 'JJ'), ('.', '.'), ('He', 'PRP'), ('did', 'VBD'), ("n't", 'RB'), ('want', 'VB'), ('to', 'TO'), ('work', 'VB'), ('at', 'IN'), ('all', 'DT'), ('and', 'CC'), ('he', 'PRP'), ('built', 'VBD'), ('his', 'PRP$'), ('house', 'NN'), ('out', 'IN'), ('of', 'IN'), ('straw', 'NN'), ('.', '.'), ('The', 'DT'), ('second', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('worked', 'VBD'), ('a', 'DT'), ('little', 'JJ'), ('bit', 'NN'), ('harder', 'RBR'), ('but', 'CC'), ('he', 'PRP'), ('was', 'VBD'), ('somewhat', 'RB'), ('lazy', 'JJ'), ('too', 'RB'), ('and', 'CC'), ('he', 'PRP'), ('built', 'VBD'), ('his', 'PRP$'), ('house', 'NN'), ('out', 'IN'), ('of', 'IN'), ('sticks', 'NNS'), ('.', '.'), ('Then', 'RB'), (',', ','), ('they', 'PRP'), ('sang', 'VBD'), ('and', 'CC'), ('danced', 'VBD'), ('and', 'CC'), ('played', 'VBD'), ('together', 'RB'), ('the', 'DT'), ('rest', 'NN'), ('of', 'IN'), ('the', 'DT'), ('day', 'NN'), ('.', '.'), ('The', 'DT'), ('third', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('worked', 'VBD'), ('hard', 'JJ'), ('all', 'DT'), ('day', 'NN'), ('and', 'CC'), ('built', 'VBD'), ('his', 'PRP$'), ('house', 'NN'), ('with', 'IN'), ('bricks', 'NNS'), ('.', '.'), ('It', 'PRP'), ('was', 'VBD'), ('a', 'DT'), ('sturdy', 'JJ'), ('house', 'NN'), ('complete', 'JJ'), ('with', 'IN'), ('a', 'DT'), ('fine', 'JJ'), ('fireplace', 'NN'), ('and', 'CC'), ('chimney', 'NN'), ('.', '.'), ('It', 'PRP'), ('looked', 'VBD'), ('like', 'IN'), ('it', 'PRP'), ('could', 'MD'), ('withstand', 'VB'), ('the', 'DT'), ('strongest', 'JJS'), ('winds', 'NNS'), ('.', '.')]

spacy POS tagging

In [21]
import spacy
nlp = spacy.load("en_core_web_sm")
sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
doc = nlp(sentence)
pos_tagging = [(token.text, token.pos_) for token in doc]
print(pos_tagging)
[('Once', 'ADV'), ('upon', 'SCONJ'), ('a', 'DET'), ('time', 'NOUN'), ('there', 'PRON'), ('was', 'VERB'), ('an', 'DET'), ('old', 'ADJ'), ('mother', 'NOUN'), ('pig', 'NOUN'), ('who', 'PRON'), ('had', 'VERB'), ('three', 'NUM'), ('little', 'ADJ'), ('pigs', 'NOUN'), ('and', 'CCONJ'), ('not', 'PART'), ('enough', 'ADJ'), ('food', 'NOUN'), ('to', 'PART'), ('feed', 'VERB'), ('them', 'PRON'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('So', 'ADV'), ('when', 'SCONJ'), ('they', 'PRON'), ('were', 'AUX'), ('old', 'ADJ'), ('enough', 'ADV'), (',', 'PUNCT'), ('she', 'PRON'), ('sent', 'VERB'), ('them', 'PRON'), ('out', 'ADP'), ('into', 'ADP'), ('the', 'DET'), ('world', 'NOUN'), ('to', 'PART'), ('seek', 'VERB'), ('their', 'PRON'), ('fortunes', 'NOUN'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('The', 'DET'), ('first', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('was', 'AUX'), ('very', 'ADV'), ('lazy', 'ADJ'), ('.', 'PUNCT'), ('He', 'PRON'), ('did', 'AUX'), ("n't", 'PART'), ('want', 'VERB'), ('to', 'PART'), ('work', 'VERB'), ('at', 'ADV'), ('all', 'ADV'), ('and', 'CCONJ'), ('he', 'PRON'), ('built', 'VERB'), ('his', 'PRON'), ('house', 'NOUN'), ('out', 'ADP'), ('of', 'ADP'), ('straw', 'NOUN'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('The', 'DET'), ('second', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('worked', 'VERB'), ('a', 'DET'), ('little', 'ADJ'), ('bit', 'NOUN'), ('harder', 'ADV'), ('but', 'CCONJ'), ('he', 'PRON'), ('was', 'AUX'), ('somewhat', 'ADV'), ('lazy', 'ADJ'), ('too', 'ADV'), ('and', 'CCONJ'), ('he', 'PRON'), ('built', 'VERB'), ('his', 'PRON'), ('house', 'NOUN'), ('out', 'ADP'), ('of', 'ADP'), ('sticks', 'NOUN'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('Then', 'ADV'), (',', 'PUNCT'), ('they', 'PRON'), ('sang', 'VERB'), ('and', 'CCONJ'), ('danced', 'VERB'), ('and', 'CCONJ'), ('played', 'VERB'), ('together', 'ADV'), ('the', 'DET'), ('rest', 'NOUN'), ('of', 'ADP'), ('the', 'DET'), ('day', 'NOUN'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('The', 'DET'), ('third', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('worked', 'VERB'), ('hard', 'ADV'), ('all', 'DET'), ('day', 'NOUN'), ('and', 'CCONJ'), ('built', 'VERB'), ('his', 'PRON'), ('house', 'NOUN'), ('with', 'ADP'), ('bricks', 'NOUN'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('It', 'PRON'), ('was', 'AUX'), ('a', 'DET'), ('sturdy', 'ADJ'), ('house', 'NOUN'), ('complete', 'ADJ'), ('with', 'ADP'), ('a', 'DET'), ('fine', 'ADJ'), ('fireplace', 'NOUN'), ('and', 'CCONJ'), ('chimney', 'NOUN'), ('.', 'PUNCT'), ('\n', 'SPACE'), ('It', 'PRON'), ('looked', 'VERB'), ('like', 'SCONJ'), ('it', 'PRON'), ('could', 'AUX'), ('withstand', 'VERB'), ('the', 'DET'), ('strongest', 'ADJ'), ('winds', 'NOUN'), ('.', 'PUNCT')]

NLTK Lemmatization

In [18]
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet
def get_wordnet_pos(treebank_tag):
        """
        return WORDNET POS compliance to WORDENT lemmatization (a,n,r,v) 
        """
        if treebank_tag.startswith('J'):
            return wordnet.ADJ
        elif treebank_tag.startswith('V'):
            return wordnet.VERB
        elif treebank_tag.startswith('N'):
            return wordnet.NOUN
        elif treebank_tag.startswith('R'):
            return wordnet.ADV
        else:
            # As default pos in lemmatization is Noun
            return wordnet.NOUN

from nltk import pos_tag
sentence = """Studying is important to pass your exam."""
# sentence = """I saw Alice, she was studying."""
lemmatizer = WordNetLemmatizer()
word_tokens = word_tokenize(sentence)
pos_tags = pos_tag(word_tokens)
lemma_tokens = [lemmatizer.lemmatize(word) for word, tag in pos_tags]
print(lemma_tokens)
lemma_tokens = [lemmatizer.lemmatize(word, pos=get_wordnet_pos(tag)) for word, tag in pos_tags]
print(lemma_tokens)
['Studying', 'is', 'important', 'to', 'pas', 'your', 'exam', '.']
['Studying', 'be', 'important', 'to', 'pass', 'your', 'exam', '.']

NLTK Lemmatization POS tagging

In [2]
from nltk.corpus import wordnet
def get_wordnet_pos(treebank_tag):
        """
        return WORDNET POS compliance to WORDENT lemmatization (a,n,r,v) 
        """
        if treebank_tag.startswith('J'):
            return wordnet.ADJ
        elif treebank_tag.startswith('V'):
            return wordnet.VERB
        elif treebank_tag.startswith('N'):
            return wordnet.NOUN
        elif treebank_tag.startswith('R'):
            return wordnet.ADV
        else:
            # As default pos in lemmatization is Noun
            return wordnet.NOUN
In [32]
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk import pos_tag
sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
lemmatizer = WordNetLemmatizer()
word_tokens = word_tokenize(sentence)
pos_tagging = pos_tag(word_tokens)

lemma_tokens = [lemmatizer.lemmatize(word, pos=get_wordnet_pos(pos_t)) for word, pos_t in pos_tagging]
print(lemma_tokens)
['Once', 'upon', 'a', 'time', 'there', 'be', 'an', 'old', 'mother', 'pig', 'who', 'have', 'three', 'little', 'pig', 'and', 'not', 'enough', 'food', 'to', 'fee', 'them', '.', 'So', 'when', 'they', 'be', 'old', 'enough', ',', 'she', 'send', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortune', '.', 'The', 'first', 'little', 'pig', 'be', 'very', 'lazy', '.', 'He', 'do', "n't", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'build', 'his', 'house', 'out', 'of', 'straw', '.', 'The', 'second', 'little', 'pig', 'work', 'a', 'little', 'bit', 'hard', 'but', 'he', 'be', 'somewhat', 'lazy', 'too', 'and', 'he', 'build', 'his', 'house', 'out', 'of', 'stick', '.', 'Then', ',', 'they', 'sing', 'and', 'dance', 'and', 'play', 'together', 'the', 'rest', 'of', 'the', 'day', '.', 'The', 'third', 'little', 'pig', 'work', 'hard', 'all', 'day', 'and', 'build', 'his', 'house', 'with', 'brick', '.', 'It', 'be', 'a', 'sturdy', 'house', 'complete', 'with', 'a', 'fine', 'fireplace', 'and', 'chimney', '.', 'It', 'look', 'like', 'it', 'could', 'withstand', 'the', 'strong', 'wind', '.']

spacy lemmatization

In [21]
import spacy
nlp = spacy.load("en_core_web_sm")
sentence = """Studying is important to pass your exam."""
# sentence = """I saw Alice, she was studying."""
doc = nlp(sentence)
lemma_tokens = [token.lemma_ for token in doc]
print(lemma_tokens)
['study', 'be', 'important', 'to', 'pass', 'your', 'exam', '.']

NLTK Noun phrase chunking

In [23]
import nltk
from nltk import Tree
from nltk.tokenize import word_tokenize
sentence = """Machine learning is a field of study in artificial intelligence"""
tokenized_text = nltk.word_tokenize(sentence)
tagged_token = nltk.pos_tag(tokenized_text)
grammer = r"""
            NP: {<DT>?<JJ>+<NN>}
            {<DT>+<NN>}
            {<NN>+}
            """
phrases = nltk.RegexpParser(grammer)
result = phrases.parse(tagged_token)
noun_phrases = []
for child in result:
    if isinstance(child, Tree):               
        if child.label() == 'NP':
            np = " ".join([lf[0] for lf in child.leaves()])
            noun_phrases.append(np)
print(noun_phrases)
['Machine learning', 'a field', 'study', 'artificial intelligence']

spacy Noun phrase chunking

In [24]
import spacy
nlp = spacy.load("en_core_web_sm")
sentence =  """Machine learning is a field of study in artificial intelligence"""
doc = nlp(sentence)
noun_phrases = [np for np in doc.noun_chunks]
print(noun_phrases)
[Machine learning, a field, study, artificial intelligence]

NLTK Dependency Parsing

In [ ]
from nltk.parse.stanford import StanfordDependencyParser
# Path to CoreNLP jar unzipped

jar_path = '/content/stanford-corenlp-4.2.2/stanford-corenlp-4.2.2.jar'

# Path to CoreNLP model jar
models_jar_path = '/content/stanford-corenlp-4.2.2-models-english.jar'

sentence = 'Deemed universities charge huge fees'

# Initialize StanfordDependency Parser from the path
parser = StanfordDependencyParser(path_to_jar = jar_path, path_to_models_jar = models_jar_path)

# Parse the sentence
result = parser.raw_parse(sentence)
dependency = result.__next__()

spacy Dependency Parsing

In [25]
import spacy
nlp = spacy.load("en_core_web_sm")
sentence = """Bob ate apple"""
doc = nlp(sentence)
for token in doc:
    print(token.text + ", " + token.dep_ + ", " + token.head.text)
Bob, nsubj, ate
ate, ROOT, ate
apple, dobj, ate

NLTK Text Preprocessing Pipeline

In [9]
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk import pos_tag
from nltk.corpus import stopwords
import string
text = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english'))
processed_text = []
sentences = sent_tokenize(text)
for sentence in sentences:
    tokens = word_tokenize(sentence)
    pos_tagged_tokens = pos_tag(tokens)
    for token, tag in pos_tagged_tokens:
        if token.lower() in stop_words or token in string.punctuation or token.isdigit():
            continue
        lemma = lemmatizer.lemmatize(token, pos=get_wordnet_pos(tag))
        processed_text.append((lemma, tag))
print(processed_text)
[('upon', 'IN'), ('time', 'NN'), ('old', 'JJ'), ('mother', 'NN'), ('pig', 'NN'), ('three', 'CD'), ('little', 'JJ'), ('pig', 'NNS'), ('enough', 'RB'), ('food', 'NN'), ('fee', 'VB'), ('old', 'JJ'), ('enough', 'RB'), ('send', 'VBD'), ('world', 'NN'), ('seek', 'VB'), ('fortune', 'NNS'), ('first', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('lazy', 'JJ'), ("n't", 'RB'), ('want', 'VB'), ('work', 'VB'), ('build', 'VBD'), ('house', 'NN'), ('straw', 'NN'), ('second', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('work', 'VBD'), ('little', 'JJ'), ('bit', 'NN'), ('hard', 'RBR'), ('somewhat', 'RB'), ('lazy', 'JJ'), ('build', 'VBD'), ('house', 'NN'), ('stick', 'NNS'), ('sing', 'VBD'), ('dance', 'VBD'), ('play', 'VBD'), ('together', 'RB'), ('rest', 'NN'), ('day', 'NN'), ('third', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('work', 'VBD'), ('hard', 'JJ'), ('day', 'NN'), ('build', 'VBD'), ('house', 'NN'), ('brick', 'NNS'), ('sturdy', 'JJ'), ('house', 'NN'), ('complete', 'JJ'), ('fine', 'JJ'), ('fireplace', 'NN'), ('chimney', 'NN'), ('look', 'VBD'), ('like', 'IN'), ('could', 'MD'), ('withstand', 'VB'), ('strong', 'JJS'), ('wind', 'NNS')]

spaCy Text Preprocessing Pipeline

In [12]
import spacy
nlp = spacy.load("en_core_web_sm")
sentence = """Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. 
So when they were old enough, she sent them out into the world to seek their fortunes. 
The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. 
The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. 
Then, they sang and danced and played together the rest of the day. 
The third little pig worked hard all day and built his house with bricks. 
It was a sturdy house complete with a fine fireplace and chimney. 
It looked like it could withstand the strongest winds."""
doc = nlp(sentence)

processed_text = []
for token in doc:
    if token.is_stop or token.is_punct or token.is_digit or token.is_space:
        continue
    processed_text.append((token.lemma_, token.pos_))

print(processed_text)
[('time', 'NOUN'), ('old', 'ADJ'), ('mother', 'NOUN'), ('pig', 'NOUN'), ('little', 'ADJ'), ('pig', 'NOUN'), ('food', 'NOUN'), ('feed', 'VERB'), ('old', 'ADJ'), ('send', 'VERB'), ('world', 'NOUN'), ('seek', 'VERB'), ('fortune', 'NOUN'), ('little', 'ADJ'), ('pig', 'NOUN'), ('lazy', 'ADJ'), ('want', 'VERB'), ('work', 'VERB'), ('build', 'VERB'), ('house', 'NOUN'), ('straw', 'NOUN'), ('second', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('work', 'VERB'), ('little', 'ADJ'), ('bit', 'NOUN'), ('hard', 'ADV'), ('somewhat', 'ADV'), ('lazy', 'ADJ'), ('build', 'VERB'), ('house', 'NOUN'), ('stick', 'NOUN'), ('sing', 'VERB'), ('dance', 'VERB'), ('play', 'VERB'), ('rest', 'NOUN'), ('day', 'NOUN'), ('little', 'ADJ'), ('pig', 'NOUN'), ('work', 'VERB'), ('hard', 'ADV'), ('day', 'NOUN'), ('build', 'VERB'), ('house', 'NOUN'), ('brick', 'NOUN'), ('sturdy', 'ADJ'), ('house', 'NOUN'), ('complete', 'ADJ'), ('fine', 'ADJ'), ('fireplace', 'NOUN'), ('chimney', 'NOUN'), ('look', 'VERB'), ('like', 'SCONJ'), ('withstand', 'VERB'), ('strong', 'ADJ'), ('wind', 'NOUN')]
In [4]
import spacy
 
#load the small English model
nlp = spacy.load("en_core_web_sm")

sentence = """The dog barked in the park on another dog.
The owner of the dog put him on the leash since he barked.
My dog is barking and chasing its tail.
"""

doc = nlp(sentence)
filtered_tokens = [token.text for token in doc if not token.is_stop and not token.is_punct]
print(set(filtered_tokens))
{'chasing', 'leash', 'barked', 'tail', 'barking', 'park', 'owner', '\n', 'dog'}
In [27]
import string
string.punctuation
Out [27] '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
In [ ]