import tensorflow as tf
from keras import backend as K
import keras.layers as layers
from keras.models import Model# Initialize session
sess = tf.Session()
K.set_session(sess)# Instantiate the elmo model
elmo_model = hub.Module("https://tfhub.dev/google/elmo/1", trainable=True)
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())# We create a function to integrate the tensorflow model with a Keras model
# This requires explicitly casting the tensor to a string, because of a Keras quirk
def ElmoEmbedding(x):return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature="default", as_dict=True)["default"]input_text = layers.Input(shape=(1,), dtype=tf.string)
embedding = layers.Lambda(ElmoEmbedding, output_shape=(1024,))(input_text)
dense = layers.Dense(256, activation='relu')(embedding)
pred = layers.Dense(1, activation='sigmoid')(dense)model = Model(inputs=[input_text], outputs=pred)model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()复制代码

# location: fastai/lm_rnn.pydef get_language_model(n_tok, emb_sz, n_hid, n_layers, pad_token,dropout=0.4, dropouth=0.3, dropouti=0.5, dropoute=0.1, wdrop=0.5, tie_weights=True, qrnn=False, bias=False):"""Returns a SequentialRNN model.A RNN_Encoder layer is instantiated using the parameters provided.This is followed by the creation of a LinearDecoder layer.Also by default (i.e. tie_weights = True), the embedding matrix used in the RNN_Encoderis used to  instantiate the weights for the LinearDecoder layer.The SequentialRNN layer is the native torch's Sequential wrapper that puts the RNN_Encoder andLinearDecoder layers sequentially in the model.Args:n_tok (int): number of unique vocabulary words (or tokens) in the source datasetemb_sz (int): the embedding size to use to encode each tokenn_hid (int): number of hidden activation per LSTM layern_layers (int): number of LSTM layers to use in the architecturepad_token (int): the int value used for padding text.dropouth (float): dropout to apply to the activations going from one LSTM layer to anotherdropouti (float): dropout to apply to the input layer.dropoute (float): dropout to apply to the embedding layer.wdrop (float): dropout used for a LSTM's internal (or hidden) recurrent weights.tie_weights (bool): decide if the weights of the embedding matrix in the RNN encoder should be tied to theweights of the LinearDecoder layer.qrnn (bool): decide if the model is composed of LSTMS (False) or QRNNs (True).bias (bool): decide if the decoder should have a bias layer or not.Returns:A SequentialRNN model"""rnn_enc = RNN_Encoder(n_tok, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token,dropouth=dropouth, dropouti=dropouti, dropoute=dropoute, wdrop=wdrop, qrnn=qrnn)enc = rnn_enc.encoder if tie_weights else Nonereturn SequentialRNN(rnn_enc, LinearDecoder(n_tok, emb_sz, dropout, tie_encoder=enc, bias=bias))def get_rnn_classifier(bptt, max_seq, n_class, n_tok, emb_sz, n_hid, n_layers, pad_token, layers, drops, bidir=False,dropouth=0.3, dropouti=0.5, dropoute=0.1, wdrop=0.5, qrnn=False):rnn_enc = MultiBatchRNN(bptt, max_seq, n_tok, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,dropouth=dropouth, dropouti=dropouti, dropoute=dropoute, wdrop=wdrop, qrnn=qrnn)return SequentialRNN(rnn_enc, PoolingLinearClassifier(layers, drops))复制代码

# location: finetune-transformer-lm/train.pydef model(X, M, Y, train=False, reuse=False):with tf.variable_scope('model', reuse=reuse):# n_special=3，作者把数据集分为三份# n_ctx 应该是 n_contextwe = tf.get_variable("we", [n_vocab+n_special+n_ctx, n_embd], initializer=tf.random_normal_initializer(stddev=0.02))we = dropout(we, embd_pdrop, train)X = tf.reshape(X, [-1, n_ctx, 2])M = tf.reshape(M, [-1, n_ctx])# 1. Embeddingh = embed(X, we)# 2. transformer blockfor layer in range(n_layer):h = block(h, 'h%d'%layer, train=train, scale=True)# 3. 计算语言模型losslm_h = tf.reshape(h[:, :-1], [-1, n_embd])lm_logits = tf.matmul(lm_h, we, transpose_b=True)lm_losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=lm_logits, labels=tf.reshape(X[:, 1:, 0], [-1]))lm_losses = tf.reshape(lm_losses, [shape_list(X)[0], shape_list(X)[1]-1])lm_losses = tf.reduce_sum(lm_losses*M[:, 1:], 1)/tf.reduce_sum(M[:, 1:], 1)# 4. 计算classifier lossclf_h = tf.reshape(h, [-1, n_embd])pool_idx = tf.cast(tf.argmax(tf.cast(tf.equal(X[:, :, 0], clf_token), tf.float32), 1), tf.int32)clf_h = tf.gather(clf_h, tf.range(shape_list(X)[0], dtype=tf.int32)*n_ctx+pool_idx)clf_h = tf.reshape(clf_h, [-1, 2, n_embd])if train and clf_pdrop > 0:shape = shape_list(clf_h)shape[1] = 1clf_h = tf.nn.dropout(clf_h, 1-clf_pdrop, shape)clf_h = tf.reshape(clf_h, [-1, n_embd])clf_logits = clf(clf_h, 1, train=train)clf_logits = tf.reshape(clf_logits, [-1, 2])clf_losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=clf_logits, labels=Y)return clf_logits, clf_losses, lm_losses复制代码