Salesforce/codet5p-110m-embedding

Hi there,

I would like to fine-tune this model on our synthetic dataset. This is my code:

model = SentenceTransformer('Salesforce/codet5p-110m-embedding', cache_folder='codet5p-110m-embedding', device='cuda', trust_remote_code=True)
raw_dataset = load_dataset('json', data_files='data.json')
train_examples = []
train_data = raw_dataset['train']
n_examples = raw_dataset['train'].num_rows
for i in tqdm(range(n_examples)):
    example = train_data[i]
    train_examples.append(InputExample(texts=[example['instruction'], example['output']]))
train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=16)
train_loss = losses.MultipleNegativesRankingLoss(model=model)
num_epochs = 3
warmup_steps = int(len(train_dataloader) * num_epochs * 0.1) #10% of train data
model.fit(
        train_objectives=[(train_dataloader, train_loss)],
        epochs=num_epochs,
        warmup_steps=warmup_steps,
        output_path='codet5p_finetuned',
        scheduler='warmupcosine',
        # use_amp=True
    )

However, it gives me the following error:

Full stack trace

```

AttributeError Traceback (most recent call last)
Cell In [14], line 1
----> 1 model.fit(
2 train_objectives=[(train_dataloader, train_loss)],
3 epochs=num_epochs,
4 warmup_steps=warmup_steps,
5 output_path='codet5p_finetuned',
6 scheduler='warmupcosine',
7 # use_amp=True
8 )

File /usr/local/lib/python3.9/dist-packages/sentence_transformers/SentenceTransformer.py:964, in SentenceTransformer.fit(self, train_objectives, evaluator, epochs, steps_per_epoch, scheduler, warmup_steps, optimizer_class, optimizer_params, weight_decay, evaluation_steps, output_path, save_best_model, max_grad_norm, use_amp, callback, show_progress_bar, checkpoint_path, checkpoint_save_steps, checkpoint_save_total_limit)
962 skip_scheduler = scaler.get_scale() != scale_before_step
963 else:
--> 964 loss_value = loss_model(features, labels)
965 loss_value.backward()
966 torch.nn.utils.clip_grad_norm_(loss_model.parameters(), max_grad_norm)

File /usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:1511, in Module._wrapped_call_impl(self, *args, **kwargs)
1509 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1510 else:
-> 1511 return self._call_impl(*args, **kwargs)

File /usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:1520, in Module._call_impl(self, *args, **kwargs)
1515 # If we don't have any hooks, we want to skip the rest of the logic in
1516 # this function, and just call forward.
1517 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1518 or _global_backward_pre_hooks or _global_backward_hooks
1519 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520 return forward_call(*args, **kwargs)
1522 try:
1523 result = None

File /usr/local/lib/python3.9/dist-packages/sentence_transformers/losses/MultipleNegativesRankingLoss.py:83, in MultipleNegativesRankingLoss.forward(self, sentence_features, labels)
82 def forward(self, sentence_features: Iterable[Dict[str, Tensor]], labels: Tensor):
---> 83 reps = [self.model(sentence_feature)["sentence_embedding"] for sentence_feature in sentence_features]
84 embeddings_a = reps[0]
85 embeddings_b = torch.cat(reps[1:])

File /usr/local/lib/python3.9/dist-packages/sentence_transformers/losses/MultipleNegativesRankingLoss.py:83, in (.0)
82 def forward(self, sentence_features: Iterable[Dict[str, Tensor]], labels: Tensor):
---> 83 reps = [self.model(sentence_feature)["sentence_embedding"] for sentence_feature in sentence_features]
84 embeddings_a = reps[0]
85 embeddings_b = torch.cat(reps[1:])

File /usr/local/lib/python3.9/dist-packages/torch/nn/modules/container.py:217, in Sequential.forward(self, input)
215 def forward(self, input):
216 for module in self:
--> 217 input = module(input)
218 return input

File /usr/local/lib/python3.9/dist-packages/sentence_transformers/models/Transformer.py:98, in Transformer.forward(self, features)
95 if "token_type_ids" in features:
96 trans_features["token_type_ids"] = features["token_type_ids"]
---> 98 output_states = self.auto_model(**trans_features, return_dict=False)
99 output_tokens = output_states[0]
101 features.update({"token_embeddings": output_tokens, "attention_mask": features["attention_mask"]})

File ~/.cache/huggingface/modules/transformers_modules/Salesforce/codet5p-110m-embedding/94f88f95672b1d4b0cc715c6011001a74f892bdd/modeling_codet5p_embedding.py:51, in CodeT5pEmbeddingModel.forward(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)
39 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
41 encoder_outputs = self.encoder(
42 input_ids=input_ids,
43 attention_mask=attention_mask,
(...)
48 return_dict=return_dict,
49 )
---> 51 embedding = F.normalize(self.proj(encoder_outputs.last_hidden_state[:, 0, :]), dim=-1)
53 return embedding

AttributeError: 'tuple' object has no attribute 'last_hidden_state'

</details>

Will appreciate any feedback.

Salesforce
/

codet5p-110m-embedding

Fine-tuning

However, it gives me the following error:

Full stack trace

```

Fine-tuning

However, it gives me the following error: Full stack trace ```

However, it gives me the following error:

Full stack trace

```