M1 pro 사용 중입니다.
VSCode에서 conda 가상환경으로
SciBERT로 파인튜닝을 진행하려고 하는데요,
device = torch.device("mps")
data=data
sentences = []
labels = []
sentence = []
label = []
for idx, row in data.iterrows():
if pd.isna(row['Token']):
if sentence:
sentences.append(sentence)
labels.append(label)
sentence = []
label = []
else:
sentence.append(row['Token'])
label.append(row['Label'])
if sentence:
sentences.append(sentence)
labels.append(label)
df_label = list(data['Label'].unique())
label2id = {label: i for i, label in enumerate(df_label)}
id2label = {i: label for i, label in enumerate(df_label)}
train_sentences, test_sentences, train_labels, test_labels = train_test_split(sentences, labels, test_size=0.2, random_state=42)
def tokenize_and_align_labels(sentences, labels, tokenizer, label2id, max_length=512):
tokenized_inputs = tokenizer(sentences, padding='max_length', truncation=True, max_length=max_length, is_split_into_words=True)
labels_aligned = []
for i, label in enumerate(labels):
word_ids = tokenized_inputs.word_ids(batch_index=i)
label_ids = []
previous_word_idx = None
for word_idx in word_ids:
if word_idx is None:
label_ids.append(-100)
elif word_idx != previous_word_idx:
label_ids.append(label2id[label[word_idx]])
else:
label_ids.append(-100)
previous_word_idx = word_idx
labels_aligned.append(label_ids)
tokenized_inputs["labels"] = labels_aligned
return tokenized_inputs
model_name = "allenai/scibert_scivocab_uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForTokenClassification.from_pretrained(model_name, num_labels=len(df_label), id2label=id2label, label2id=label2id).to(device)
epoch_range = [2, 3, 4, 5]
learning_rate_range = [5e-6, 1e-5, 2e-5, 5e-5]
batch_size = 32
performance_results = []
for lr in learning_rate_range:
for epoch in epoch_range:
print(f'Learning Rate: {lr}, Epoch: {epoch}')
train_dataset = Dataset.from_dict(tokenize_and_align_labels(train_sentences, train_labels, tokenizer, label2id))
test_dataset = Dataset.from_dict(tokenize_and_align_labels(test_sentences, test_labels, tokenizer, label2id))
train_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
test_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
training_args = TrainingArguments(
output_dir='./results',
evaluation_strategy="epoch",
learning_rate=lr,
per_device_train_batch_size=batch_size,
per_device_eval_batch_size=batch_size,
num_train_epochs=epoch,
weight_decay=0.01,
logging_dir='./logs',
logging_steps=10,
)
trainer = Trainer(
model=model.to(device),
args=training_args,
train_dataset=train_dataset,
eval_dataset=test_dataset,
)
trainer.train()
true_labels = []
pred_labels = []
for sentence, label in zip(test_sentences, test_labels):
inputs = tokenizer(sentence, is_split_into_words=True, return_tensors="pt", truncation=True, max_length=512).to(device)
outputs = model(**inputs).logits
predictions = torch.argmax(outputs, dim=2).cpu()
tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0].tolist())
pred_sentence_labels = ['O'] * len(sentence)
word_ids = inputs.word_ids()
for i, word_id in enumerate(word_ids):
if word_id is not None and word_id < len(sentence):
pred_sentence_labels[word_id] = model.config.id2label[predictions[0][i].item()]
true_labels.extend(label)
pred_labels.extend(pred_sentence_labels)
report = classification_report(true_labels, pred_labels, labels=df_label, output_dict=True)
report_df = pd.DataFrame(report).transpose()
performance_results.append({
'Learning Rate': lr,
'Epoch': epoch,
'Precision': report_df.loc['macro avg']['precision'],
'Recall': report_df.loc['macro avg']['recall'],
'F1-Score': report_df.loc['macro avg']['f1-score']
})
위 코드에서
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In[24], line 32
13 training_args = TrainingArguments(
14 output_dir='./results',
15 evaluation_strategy="epoch",
(...)
22 logging_steps=10,
23 )
25 trainer = Trainer(
26 model=model.to(device),
27 args=training_args,
28 train_dataset=train_dataset,
29 eval_dataset=test_dataset,
30 )
---> 32 trainer.train()
34 true_labels = []
35 pred_labels = []
File /opt/anaconda3/envs/bert_ner/lib/python3.8/site-packages/transformers/trainer.py:1932, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
1930 hf_hub_utils.enable_progress_bars()
1931 else:
-> 1932 return inner_training_loop(
1933 args=args,
1934 resume_from_checkpoint=resume_from_checkpoint,
...
2231 # remove once script supports set_grad_enabled
2232 _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2233 return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
RuntimeError: Placeholder storage has not been allocated on MPS device!
Output is truncated. View as a scrollable element or open in a text editor. Adjust cell output settings...
이와 같은 오류가 발생합니다. 추가로 어떤 설정을 해줘야 할까요
도움주시면 감사하겠습니다ㅠㅠ
를 한 번 눌러주시길 부탁드려요!