from datasets import load_dataset#?Prepare data?and splits
tomatoes = load_dataset("rotten_tomatoes")
train_data, test_data = tomatoes["train"], tomatoes["test"]

from transformers import AutoTokenizer, AutoModelForSequenceClassification#?Load?model and tokenizer
model_id =?"bert-base-cased"
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2
)
tokenizer = AutoTokenizer.from_pretrained(model_id)?

from transformers import DataCollatorWithPadding# Pad to?the?longest sequence?in?the?batch
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
def preprocess_function(examples):"""Tokenize?input data"""return tokenizer(examples["text"], truncation=True)
# Tokenize?train/test?data
tokenized_train = train_data.map(preprocess_function, batched=True)
tokenized_test = test_data.map(preprocess_function, batched=True)

import numpy?as?np
from datasets import load_metricdef compute_metrics(eval_pred):"""Calculate F1?score"""logits, labels =?eval_predpredictions = np.argmax(logits, axis=-1)load_f1 = load_metric("f1")f1 = load_f1.compute(predictions=predictions,?references=labels)["f1"]return?{"f1": f1}

from transformers import TrainingArguments, Trainer# Training arguments?for parameter?tuning
training_args = TrainingArguments("model",learning_rate=2e-5,per_device_train_batch_size=16,per_device_eval_batch_size=16,num_train_epochs=1,weight_decay=0.01,save_strategy="epoch",report_to="none"
)# Trainer which?executes?the?training process
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator,compute_metrics=compute_metrics,
)

from transformers import TrainingArguments, Trainer# Trainer which?executes?the?training process
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator,compute_metrics=compute_metrics,
)trainer.train()

#?Load?modelmodel_id =?"bert-base-cased"
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2
)tokenizer = AutoTokenizer.from_pretrained(model_id)
# Encoder block 11 starts?at?index?165?and
# we freeze everything before?that block
for index, (name, param) in enumerate(model.named_parameters()):if index?<?165:param.requires_grad =?False# Trainer which?executes?the?training process
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator,compute_metrics=compute_metrics,
)trainer.train()

?

from setfit import sample_dataset# We simulate a few-shot setting by sampling?16?examples per?class
sampled_train_data = sample_dataset(tomatoes["train"], num_samples=16)

from setfit import SetFitModel#?Load?a pretrained SentenceTransformer model
model = SetFitModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")

from setfit import TrainingArguments as SetFitTrainingArguments
from setfit import Trainer as?SetFitTrainer#?Define?training arguments
args = SetFitTrainingArguments(num_epochs=3, # The number of epochs?to?use for contrastive?learningnum_iterations=20 ?# The number of?text?pairs?to?generate
)args.eval_strategy = args.evaluation_strategy# Create?trainer
trainer = SetFitTrainer(model=model,args=args,train_dataset=sampled_train_data,eval_dataset=test_data,metric="f1"
)

from transformers import AutoTokenizer, AutoModelForMaskedLM#?Load?model for masked language modeling?(MLM)
model = AutoModelForMaskedLM.from_pretrained("bert-base-cased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

def preprocess_function(examples):return tokenizer(examples["text"], truncation=True)#?Tokenize?data
tokenized_train = train_data.map(preprocess_function, batched=True)
tokenized_train = tokenized_train.remove_columns("label")
tokenized_test = test_data.map(preprocess_function, batched=True)
tokenized_test = tokenized_test.remove_columns("label")

?

from transformers import DataCollatorForLanguageModeling#?Masking?Tokens
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer,mlm=True,mlm_probability=0.15
)

# Training arguments?for parameter?tuning
training_args = TrainingArguments("model",learning_rate=2e-5,per_device_train_batch_size=16,per_device_eval_batch_size=16,num_train_epochs=10,weight_decay=0.01,save_strategy="epoch",report_to="none"
)# Initialize?Trainer
trainer = Trainer(model=model,args=training_args,train_dataset=tokenized_train,eval_dataset=tokenized_test,tokenizer=tokenizer,data_collator=data_collator
)

# Save pre-trained tokenizer
tokenizer.save_pretrained("mlm")
#?Train model
trainer.train()
#?Save updated model
model.save_pretrained("mlm")

from transformers import pipeline#?Load?and?create predictions
mask_filler = pipeline("fill-mask", model="bert-base-cased")
preds = mask_filler("What a?horrible?[MASK]!")
#?Print results
for pred?in?preds:print(f">>> {pred["sequence"]}")

#?Load?and?create predictionsmask_filler = pipeline("fill-mask", model="mlm")
preds = mask_filler("What a?horrible?[MASK]!")
#?Print results
for pred?in?preds:print(f">>> {pred["sequence"]}")

from transformers import AutoModelForSequenceClassification# Fine-tune for classification
model = AutoModelForSequenceClassification.from_pretrained("mlm", num_labels=2)
tokenizer = AutoTokenizer.from_pretrained("mlm")

def align_labels(examples):token_ids = tokenizer(examples["tokens"],truncation=True,is_split_into_words=True)labels = examples["ner_tags"]updated_labels =?[]for index, label in enumerate(labels):#?Map?tokens?to?their respective wordword_ids = token_ids.word_ids(batch_index=index)previous_word_idx =?Nonelabel_ids =?[]for word_idx in word_ids:# The?start?of a?new wordif word_idx?!= previous_word_idx:previous_word_idx = word_idxupdated_label =?-100 if word_idx is None else label[word_idx]label_ids.append(updated_label)# Special?token?is?-100elif word_idx is Nonelabel_ids.append(-100)# If the?label?is B-XXX we?change?it?to I-XXXelse:updated_label = label[word_idx]if updated_label %?2?==?1:updated_label +=?1label_ids.append(updated_label)updated_labels.append(label_ids)token_ids["labels"] = updated_labelsreturn token_idstokenized = dataset.map(align_labels, batched=True)

#?Difference between original and updated?labels
print(f"Original: {example["ner_tags"]}")
print(f"Updated: {tokenized["train"][848]["labels"]}")

import evaluate#?Load sequential evaluation
seqeval = evaluate.load("seqeval")def compute_metrics(eval_pred):# Create predictionslogits, labels =?eval_predpredictions = np.argmax(logits, axis=2)true_predictions =?[]true_labels =?[]#?Document-level?iterationfor prediction, label in zip(predictions,?labels):# Token-level?iterationfor token_prediction, token_label in zip(prediction, label):# We?ignore special?tokensif token_label?!=?-100:true_predictions.append([id2label[token_prediction]])true_labels.append([id2label[token_label]])results = seqeval.compute(predictions=true_predictions, references=true_labels)return {"f1":?results["overall_f1"]}