Feature Engineering
ipynb12 kBPython · 17 cellsExtra material of chapter 5
Count Vectorization using Sklearn
from sklearn.feature_extraction.text import CountVectorizer
sents = ['The dog barked in the park on another dog',
'The owner of the dog put him on the leash since he barked.',
'My dog is barking and chasing its tail.']
cv = CountVectorizer()
X = cv.fit_transform(sents)
X = X.toarray()
print(sorted(cv.vocabulary_.keys()))
print(X)['and', 'another', 'barked', 'barking', 'chasing', 'dog', 'he', 'him', 'in', 'is', 'its', 'leash', 'my', 'of', 'on', 'owner', 'park', 'put', 'since', 'tail', 'the'] [[0 1 1 0 0 2 0 0 1 0 0 0 0 0 1 0 1 0 0 0 2] [0 0 1 0 0 1 1 1 0 0 0 1 0 1 1 1 0 1 1 0 3] [1 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0]]
Count Vectorization without Stopwords using Sklearn
from sklearn.feature_extraction.text import CountVectorizer
sents = ['The dog barked in the park on another dog',
'The owner of the dog put him on the leash since he barked.',
'My dog is barking and chasing its tail.']
cv = CountVectorizer(stop_words='english')
X = cv.fit_transform(sents)
X = X.toarray()
print(sorted(cv.vocabulary_.keys()))
print(X)['barked', 'barking', 'chasing', 'dog', 'leash', 'owner', 'park', 'tail'] [[1 0 0 2 0 0 1 0] [1 0 0 1 1 1 0 0] [0 1 1 1 0 0 0 1]]
N-Gram Count Vectorization without Stopwords using Sklearn
from sklearn.feature_extraction.text import CountVectorizer
sents = ['The dog barked in the park on another dog',
'The owner of the dog put him on the leash since he barked.',
'My dog is barking and chasing its tail.']
cv = CountVectorizer(ngram_range=(1,2), stop_words='english')
X = cv.fit_transform(sents)
X = X.toarray()
print(sorted(cv.vocabulary_.keys()))
print(X)['barked', 'barked park', 'barking', 'barking chasing', 'chasing', 'chasing tail', 'dog', 'dog barked', 'dog barking', 'dog leash', 'leash', 'leash barked', 'owner', 'owner dog', 'park', 'park dog', 'tail'] [[1 1 0 0 0 0 2 1 0 0 0 0 0 0 1 1 0] [1 0 0 0 0 0 1 0 0 1 1 1 1 1 0 0 0] [0 0 1 1 1 1 1 0 1 0 0 0 0 0 0 0 1]]
TF-IDF Vectorization using Sklearn
from sklearn.feature_extraction.text import TfidfVectorizer
sents = ['The dog barked in the park on another dog',
'The owner of the dog put him on the leash since he barked.',
'My dog is barking and chasing its tail.']
cv = TfidfVectorizer(stop_words='english')
X = cv.fit_transform(sents)
print(X.toarray())
import pandas as pd
df = pd.DataFrame(X[0].T.todense(), index=cv.get_feature_names(), columns=["TF-IDF"])
df = df.sort_values('TF-IDF', ascending=False)
print()
print(df)[[0.44102652 0. 0. 0.68499287 0. 0.
0.57989687 0. ]
[0.44451431 0. 0. 0.34520502 0.5844829 0.5844829
0. 0. ]
[0. 0.54645401 0.54645401 0.32274454 0. 0.
0. 0.54645401]]
TF-IDF
dog 0.684993
park 0.579897
barked 0.441027
barking 0.000000
chasing 0.000000
leash 0.000000
owner 0.000000
tail 0.000000
Word2Vec: using Pre-trained model in Gensim
from gensim import models
w2v = models.KeyedVectors.load_word2vec_format(
'./GoogleNews-vectors-negative300.bin', binary=True)
vect = w2v['healthy']
w2v.most_similar('happy')---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
<ipython-input-17-e64bb8329dbd> in <module>()
----> 1 from gensim import models
2 w2v = models.KeyedVectors.load_word2vec_format(
3 './GoogleNews-vectors-negative300.bin', binary=True)
4
5 vect = w2v['healthy']
ModuleNotFoundError: No module named 'gensim'from gensim import models
sents = ['The dog barked in the park on another dog',
'The owner of the dog put him on the leash since he barked.',
'My dog is barking and chasing its tail.']
# Word2vec requires the training dataset in form of a list of lists
# of tokenized sentences, so we’ll preprocess and convert sents to
sents = [sent.split() for sent in sents]
custom_model = models.Word2Vec(sents, min_count=1,vector_size=300,workers=4)
vect = custom_model.wv['dog']
print(vect[:5])
result = custom_model.wv.most_similar('barked')
print(result)[-1.7874241e-04 7.8810059e-05 1.7011166e-03 3.0030911e-03
-3.1009833e-03]
[('and', 0.11018942296504974), ('leash', 0.09745844453573227), ('him', 0.08689301460981369), ('barking', 0.07596393674612045), ('he', 0.07124919444322586), ('put', 0.06887509673833847), ('owner', 0.053101178258657455), ('its', 0.03030511364340782), ('dog', 0.026307500898838043), ('is', 0.023843316361308098)]
FastText: Train our model
from gensim import models
sents = ['The dog barked in the park on another dog',
'The owner of the dog put him on the leash since he barked',
'My dog is barking and chasing its tail']
sents = [sent.split() for sent in sents]
model = models.FastText(sentences=sents, vector_size=100, window=5, min_count=1, workers=4, sg=1)
word_embedding = model.wv['dog']
print(word_embedding[:5])
# Most similar words to a given word
similar_words = model.wv.most_similar('barked')
print("Most similar words to 'barked':", similar_words)---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[1], line 1
----> 1 from gensim import models
3 sents = ['The dog barked in the park on another dog',
4 'The owner of the dog put him on the leash since he barked',
5 'My dog is barking and chasing its tail']
7 sents = [sent.split() for sent in sents]
ModuleNotFoundError: No module named 'gensim'from gensim.test.utils import common_texts
# Example corpus (replace with your own corpus)
corpus = common_textscorpusOut [6] [['human', 'interface', 'computer'], ['survey', 'user', 'computer', 'system', 'response', 'time'], ['eps', 'user', 'interface', 'system'], ['system', 'human', 'system', 'eps'], ['user', 'response', 'time'], ['trees'], ['graph', 'trees'], ['graph', 'minors', 'trees'], ['graph', 'minors', 'survey']]
FastText?