File size: 1,044 Bytes
a2c0551
 
b022cb9
a2c0551
 
ba986c0
 
 
 
b022cb9
 
 
 
 
 
 
 
 
 
 
 
 
a2c0551
 
 
 
 
ba986c0
 
a2c0551
 
 
 
 
 
d64b893
 
 
a2c0551
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
"use server"

import { LLMPredictionFunctionParams } from "@/types"
import Groq from "groq-sdk"

export async function predict({
  systemPrompt,
  userPrompt,
  nbMaxNewTokens,
  llmVendorConfig
}: LLMPredictionFunctionParams): Promise<string> {
  const groqApiKey = `${
    llmVendorConfig.apiKey ||
    process.env.AUTH_GROQ_API_KEY ||
    ""
  }`
  const groqApiModel = `${
    llmVendorConfig.modelId ||
    process.env.LLM_GROQ_API_MODEL ||
    "mixtral-8x7b-32768"
  }`

  const groq = new Groq({
    apiKey: groqApiKey,
  })

  const messages: Groq.Chat.Completions.CompletionCreateParams.Message[] = [
    { role: "system", content: systemPrompt },
    { role: "user", content: userPrompt },
  ]

  try {
    const res = await groq.chat.completions.create({
      messages: messages,
      model: groqApiModel,
      stream: false,
      temperature: 0.5,
      max_tokens: nbMaxNewTokens,
    })

    return res.choices[0].message.content || ""
  } catch (err) {
    console.error(`error during generation: ${err}`)
    return ""
  }
}