目录
介绍：
搭建上下文或预测目标词来学习词向量
建模1：
建模2：
预测：
介绍：
Word2Vec是一种用于将文本转换为向量表示的技术。它是由谷歌团队于2013年提出的一种神经网络模型。Word2Vec可以将单词表示为高维空间中的向量，使得具有相似含义的单词在向量空间中距离较近。这种向量表示可以用于各种自然语言处理任务，如语义相似度计算、文本分类和命名实体识别等。Word2Vec的核心思想是通过预测上下文或预测目标词来学习词向量。具体而言，它使用连续词袋（CBOW）和跳字模型（Skip-gram）来训练神经网络，从而得到单词的向量表示。这些向量可以捕捉到单词之间的语义和语法关系，使得它们在计算机中更容易处理和比较。Word2Vec已经成为自然语言处理领域中常用的工具，被广泛应用于各种文本分析和语义理解任务中。
import os
os.environ['KMP_DUPLICATE_LIB_OK']='True'#Dataset 10 sentences to create word vectorscorpus = ['king is a strong man', 'queen is a wise woman', 'boy is a young man','girl is a young woman','prince is a young king','princess is a young queen','man is strong', 'woman is pretty','prince is a boy will be king','princess is a girl will be queen']#Remove stop wordsdef remove_stop_words(corpus):stop_words = ['is', 'a', 'will', 'be']results = []for text in corpus:tmp = text.split(' ')for stop_word in stop_words:if stop_word in tmp:tmp.remove(stop_word)results.append(" ".join(tmp))return resultscorpus = remove_stop_words(corpus)corpus'''结果：
['king strong man','queen wise woman','boy young man','girl young woman','prince young king','princess young queen','man strong','woman pretty','prince boy king','princess girl queen']
'''搭建上下文或预测目标词来学习词向量
words = []
for text in corpus:for word in text.split(' '):words.append(word)words = set(words)word2int = {}for i,word in enumerate(words):word2int[word] = iprint(word2int)
'''结果：
{'strong': 0,'wise': 1,'man': 2,'boy': 3,'queen': 4,'king': 5,'princess': 6,'young': 7,'woman': 8,'pretty': 9,'prince': 10,'girl': 11}
'''sentences = []
for sentence in corpus:sentences.append(sentence.split())print(sentences)WINDOW_SIZE = 2#距离为2data = []
for sentence in sentences:for idx, word in enumerate(sentence):for neighbor in sentence[max(idx - WINDOW_SIZE, 0) : min(idx + WINDOW_SIZE, len(sentence)) + 1] : if neighbor != word:data.append([word, neighbor])print(data)data
'''结果：
[['king', 'strong'],['king', 'man'],['strong', 'king'],['strong', 'man'],['man', 'king'],['man', 'strong'],['queen', 'wise'],['queen', 'woman'],['wise', 'queen'],['wise', 'woman'],['woman', 'queen'],['woman', 'wise'],['boy', 'young'],['boy', 'man'],['young', 'boy'],['young', 'man'],['man', 'boy'],['man', 'young'],['girl', 'young'],['girl', 'woman'],['young', 'girl'],['young', 'woman'],['woman', 'girl'],['woman', 'young'],['prince', 'young'],['prince', 'king'],['young', 'prince'],['young', 'king'],['king', 'prince'],['king', 'young'],['princess', 'young'],['princess', 'queen'],['young', 'princess'],['young', 'queen'],['queen', 'princess'],['queen', 'young'],['man', 'strong'],['strong', 'man'],['woman', 'pretty'],['pretty', 'woman'],['prince', 'boy'],['prince', 'king'],['boy', 'prince'],['boy', 'king'],['king', 'prince'],['king', 'boy'],['princess', 'girl'],['princess', 'queen'],['girl', 'princess'],['girl', 'queen'],['queen', 'princess'],['queen', 'girl']]
'''搭建X,Y
import pandas as pd
for text in corpus:print(text)df = pd.DataFrame(data, columns = ['input', 'label'])word2int#Define Tensorflow Graph
import tensorflow as tf
import numpy as npONE_HOT_DIM = len(words)# function to convert numbers to one hot vectors
def to_one_hot_encoding(data_point_index):one_hot_encoding = np.zeros(ONE_HOT_DIM)one_hot_encoding[data_point_index] = 1return one_hot_encodingX = [] # input word
Y = [] # target wordfor x, y in zip(df['input'], df['label']):X.append(to_one_hot_encoding(word2int[ x ]))Y.append(to_one_hot_encoding(word2int[ y ]))# convert them to numpy arrays
X_train = np.asarray(X)
Y_train = np.asarray(Y)建模1：
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()# making placeholders for X_train and Y_train
x = tf.placeholder(tf.float32, shape=(None, ONE_HOT_DIM))
y_label = tf.placeholder(tf.float32, shape=(None, ONE_HOT_DIM))# word embedding will be 2 dimension for 2d visualization
EMBEDDING_DIM = 2 # hidden layer: which represents word vector eventually
W1 = tf.Variable(tf.random_normal([ONE_HOT_DIM, EMBEDDING_DIM]))
b1 = tf.Variable(tf.random_normal([1])) #bias
hidden_layer = tf.add(tf.matmul(x,W1), b1)# output layer
W2 = tf.Variable(tf.random_normal([EMBEDDING_DIM, ONE_HOT_DIM]))
b2 = tf.Variable(tf.random_normal([1]))
prediction = tf.nn.softmax(tf.add( tf.matmul(hidden_layer, W2), b2))# loss function: cross entropy
loss = tf.reduce_mean(-tf.reduce_sum(y_label * tf.log(prediction), axis=[1]))# training operation
train_op = tf.train.GradientDescentOptimizer(0.05).minimize(loss)sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init) iteration = 20000
for i in range(iteration):# input is X_train which is one hot encoded word# label is Y_train which is one hot encoded neighbor wordsess.run(train_op, feed_dict={x: X_train, y_label: Y_train})if i % 3000 == 0:print('iteration '+str(i)+' loss is : ', sess.run(loss, feed_dict={x: X_train, y_label: Y_train}))# Now the hidden layer (W1 + b1) is actually the word look up table
vectors = sess.run(W1 + b1)
print(vectors)import matplotlib.pyplot as pltfig, ax = plt.subplots()for word, x1, x2 in zip(words, w2v_df['x1'], w2v_df['x2']):ax.annotate(word, (x1,x2 ))PADDING = 1.0
x_axis_min = np.amin(vectors, axis=0)[0] - PADDING
y_axis_min = np.amin(vectors, axis=0)[1] - PADDING
x_axis_max = np.amax(vectors, axis=0)[0] + PADDING
y_axis_max = np.amax(vectors, axis=0)[1] + PADDINGplt.xlim(x_axis_min,x_axis_max)
plt.ylim(y_axis_min,y_axis_max)
plt.rcParams["figure.figsize"] = (10,10)plt.show()
建模2：
# Deep learning:
from tensorflow.python.keras.models import Input
from keras.models import Model
from keras.layers import Dense# Defining the size of the embedding
embed_size = 2# Defining the neural network
#inp = Input(shape=(X.shape[1],))
#x = Dense(units=embed_size, activation='linear')(inp)
#x = Dense(units=Y.shape[1], activation='softmax')(x)
xx = Input(shape=(X_train.shape[1],))
yy = Dense(units=embed_size, activation='linear')(xx)
yy = Dense(units=Y_train.shape[1], activation='softmax')(yy)
model = Model(inputs=xx, outputs=yy)
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam')# Optimizing the network weights
model.fit(x=X_train, y=Y_train, batch_size=256,epochs=1000)# Obtaining the weights from the neural network.
# These are the so called word embeddings# The input layer
weights = model.get_weights()[0]# Creating a dictionary to store the embeddings in. The key is a unique word and
# the value is the numeric vector
embedding_dict = {}
for word in words: embedding_dict.update({word: weights[df.get(word)]})import matplotlib.pyplot as pltfig, ax = plt.subplots()#for word, x1, x2 in zip(words, w2v_df['x1'], w2v_df['x2']):
for word, x1, x2 in zip(words, weights[:,0], weights[:,1]):ax.annotate(word, (x1,x2 ))PADDING = 1.0
x_axis_min = np.amin(vectors, axis=0)[0] - PADDING
y_axis_min = np.amin(vectors, axis=0)[1] - PADDING
x_axis_max = np.amax(vectors, axis=0)[0] + PADDING
y_axis_max = np.amax(vectors, axis=0)[1] + PADDINGplt.xlim(x_axis_min,x_axis_max)
plt.ylim(y_axis_min,y_axis_max)
plt.rcParams["figure.figsize"] = (10,10)plt.show()
预测：
X_train[2]
#结果：array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) strongmodel.predict(X_train)[2]
'''结果：
array([0.07919139, 0.0019384 , 0.48794392, 0.05578128, 0.00650001,0.10083131, 0.02451131, 0.03198219, 0.04424168, 0.0013569 ,0.16189449, 0.00382716], dtype=float32) 预测结果：man
'''
