Custom Prompts

prompt = """
You are an expert medical school professor specializing in grading students' answers to questions.
You are grading the following question:
{question}
Here is the real answer:
{ground_truth}
You are grading the following predicted answer:
{response}
"""

# Create a list of choices
choices = ["Correct", "Correct but Incomplete", "Incorrect"]

# Create scores for the choices
choice_scores = [1.0, 0.5, 0.0]

data = [{
      "user_question": "What causes diabetes?",
      "ground_truth_response": "Diabetes is a metabolic disorder characterized by high blood sugar levels. It is primarily caused by a combination of genetic and environmental factors, including obesity and lack of physical activity.",
      "user_response": "Diabetes is primarily caused by a combination of genetic and environmental factors, including obesity and lack of physical activity."
}]

prompt_var_to_column_mapping = {
    "question": "user_question",
    "ground_truth": "ground_truth_response",
    "response": "user_response"
}

from uptrain import CustomPromptEval, EvalLLM, Settings
import json

OPENAI_API_KEY = "sk-*****************"  # Insert your OpenAI key here
eval_llm = EvalLLM(settings=Settings(openai_api_key=OPENAI_API_KEY, response_format={"type":"json_object"}))

results = eval_llm.evaluate(
    data = data,
    checks = [CustomPromptEval(
        prompt = prompt,
        choices = choices,
        choice_scores = choice_scores,
        prompt_var_to_column_mapping = prompt_var_to_column_mapping
    )]
)    

[
   {
      "Choice": "CORRECT BUT INCOMPLETE",
      "Explanation": "The predicted answer correctly identifies the primary causes of diabetes as genetic and environmental factors, including obesity and lack of physical activity. However, it does not mention that diabetes is a metabolic disorder characterized by high blood sugar levels, which is an important aspect of the real answer.",
      "score_custom_prompt": 0.5
   }
]