from datasets import load_dataset# Prepare data and splits
tomatoes = load_dataset("rotten_tomatoes")
train_data, test_data = tomatoes["train"], tomatoes["test"]

from transformers import AutoTokenizer, AutoModelForSequenceClassification# Load model and tokenizer
model_id = "bert-base-cased"
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2
)
tokenizer = AutoTokenizer.from_pretrained(model_id) 

from transformers import DataCollatorWithPadding# Pad to the longest sequence in the batch
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
def preprocess_function(examples):"""Tokenize input data"""return tokenizer(examples["text"], truncation=True)
# Tokenize train/test data
tokenized_train = train_data.map(preprocess_function, batched=True)
tokenized_test = test_data.map(preprocess_function, batched=True)

import numpy as np
from datasets import load_metricdef compute_metrics(eval_pred):"""Calculate F1 score"""logits, labels = eval_predpredictions = np.argmax(logits, axis=-1)load_f1 = load_metric("f1")f1 = load_f1.compute(predictions=predictions, references=labels)["f1"]return {"f1": f1}

from transformers import TrainingArguments, Trainer# Training arguments for parameter tuning
training_args = TrainingArguments("model",learning_rate=2e-5,per_device_train_batch_size=16,per_device_eval_batch_size=16,num_train_epochs=1,weight_decay=0.01,save_strategy="epoch",report_to="none"
)# Trainer which executes the training process
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator,compute_metrics=compute_metrics,
)

from transformers import TrainingArguments, Trainer# Trainer which executes the training process
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator,compute_metrics=compute_metrics,
)trainer.train()

# Load modelmodel_id = "bert-base-cased"
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2
)tokenizer = AutoTokenizer.from_pretrained(model_id)
# Encoder block 11 starts at index 165 and
# we freeze everything before that block
for index, (name, param) in enumerate(model.named_parameters()):if index < 165:param.requires_grad = False# Trainer which executes the training process
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator,compute_metrics=compute_metrics,
)trainer.train()

​

from setfit import sample_dataset# We simulate a few-shot setting by sampling 16 examples per class
sampled_train_data = sample_dataset(tomatoes["train"], num_samples=16)

from setfit import SetFitModel# Load a pretrained SentenceTransformer model
model = SetFitModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")

from setfit import TrainingArguments as SetFitTrainingArguments
from setfit import Trainer as SetFitTrainer# Define training arguments
args = SetFitTrainingArguments(num_epochs=3, # The number of epochs to use for contrastive learningnum_iterations=20  # The number of text pairs to generate
)args.eval_strategy = args.evaluation_strategy# Create trainer
trainer = SetFitTrainer(model=model,args=args,train_dataset=sampled_train_data,eval_dataset=test_data,metric="f1"
)

from transformers import AutoTokenizer, AutoModelForMaskedLM# Load model for masked language modeling (MLM)
model = AutoModelForMaskedLM.from_pretrained("bert-base-cased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

def preprocess_function(examples):return tokenizer(examples["text"], truncation=True)# Tokenize data
tokenized_train = train_data.map(preprocess_function, batched=True)
tokenized_train = tokenized_train.remove_columns("label")
tokenized_test = test_data.map(preprocess_function, batched=True)
tokenized_test = tokenized_test.remove_columns("label")

​

from transformers import DataCollatorForLanguageModeling# Masking Tokens
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer,mlm=True,mlm_probability=0.15
)

# Training arguments for parameter tuning
training_args = TrainingArguments("model",learning_rate=2e-5,per_device_train_batch_size=16,per_device_eval_batch_size=16,num_train_epochs=10,weight_decay=0.01,save_strategy="epoch",report_to="none"
)# Initialize Trainer
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator
)

# Save pre-trained tokenizer
tokenizer.save_pretrained("mlm")
# Train model
trainer.train()
# Save updated model
model.save_pretrained("mlm")

from transformers import pipeline# Load and create predictions
mask_filler = pipeline("fill-mask", model="bert-base-cased")
preds = mask_filler("What a horrible [MASK]!")
# Print results
for pred in preds:print(f">>> {pred["sequence"]}")

# Load and create predictionsmask_filler = pipeline("fill-mask", model="mlm")
preds = mask_filler("What a horrible [MASK]!")
# Print results
for pred in preds:print(f">>> {pred["sequence"]}")

from transformers import AutoModelForSequenceClassification# Fine-tune for classification
model = AutoModelForSequenceClassification.from_pretrained("mlm", num_labels=2)
tokenizer = AutoTokenizer.from_pretrained("mlm")

def align_labels(examples):token_ids = tokenizer(examples["tokens"],truncation=True,is_split_into_words=True)labels = examples["ner_tags"]updated_labels = []for index, label in enumerate(labels):# Map tokens to their respective wordword_ids = token_ids.word_ids(batch_index=index)previous_word_idx = Nonelabel_ids = []for word_idx in word_ids:# The start of a new wordif word_idx != previous_word_idx:previous_word_idx = word_idxupdated_label = -100 if word_idx is None else label[word_idx]label_ids.append(updated_label)# Special token is -100elif word_idx is Nonelabel_ids.append(-100)# If the label is B-XXX we change it to I-XXXelse:updated_label = label[word_idx]if updated_label % 2 == 1:updated_label += 1label_ids.append(updated_label)updated_labels.append(label_ids)token_ids["labels"] = updated_labelsreturn token_idstokenized = dataset.map(align_labels, batched=True)

# Difference between original and updated labels
print(f"Original: {example["ner_tags"]}")
print(f"Updated: {tokenized["train"][848]["labels"]}")

import evaluate# Load sequential evaluation
seqeval = evaluate.load("seqeval")def compute_metrics(eval_pred):# Create predictionslogits, labels = eval_predpredictions = np.argmax(logits, axis=2)true_predictions = []true_labels = []# Document-level iterationfor prediction, label in zip(predictions, labels):# Token-level iterationfor token_prediction, token_label in zip(prediction, label):# We ignore special tokensif token_label != -100:true_predictions.append([id2label[token_prediction]])true_labels.append([id2label[token_label]])results = seqeval.compute(predictions=true_predictions, references=true_labels)return {"f1": results["overall_f1"]}