Guideline Adherence

from uptrain import EvalLLM, ConversationGuidelineAdherence

OPENAI_API_KEY = "sk-********************"  # Insert your OpenAI key here

data = [{
    'conversation' : [
        {"role": "patient", "content": "Help"}, 
        {"role": "nurse", "content": "what do you need"}, 
        {"role": "patient", "content": "Having chest pain"}, 
        {"role": "nurse", "content": "Sorry, I am not sure what that means"},
        {"role": "patient", "content": "You don't understand. Do something! I am having severe pain in my chest"}
    ]  
}]

eval_llm = EvalLLM(openai_api_key=OPENAI_API_KEY)

res = eval_llm.evaluate(
    data=data,
    checks=[ConversationGuidelineAdherence(guideline="Provide emergency contact information if the patient is in distress", guideline_name="Emergency Contact Information")],
)

{
       "score_conversation_Emergency Contact Information_adherence": 0.0,
      "explanation_conversation_Emergency Contact Information_adherence": "[\"The given conversation strictly violates the guideline of providing emergency contact information if the patient is in distress. Despite the patient clearly expressing distress by mentioning 'Having chest pain' and emphasizing the severity of the situation with 'I am having severe pain in my chest', the nurse fails to provide any emergency contact information. Instead, the nurse responds with uncertainty and does not take appropriate action to address the patient's distress. This violation of the guideline puts the patient at risk by not providing the necessary support in a critical situation.\"]"
}