# pylint: disable=line-too-long,useless-suppression
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------

"""
DESCRIPTION:
    Given an AIProjectClient, this sample demonstrates how to use the synchronous
    `.evaluators` methods to create, get and list evaluators.

USAGE:
    python sample_prompt_based_custom_evaluators.py

    Before running the sample:

    pip install "azure-ai-projects>=2.0.0b1" azure-identity python-dotenv

    Set these environment variables with your own values:
    1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your
       Microsoft Foundry project. It has the form: https://<account_name>.services.ai.azure.com/api/projects/<project_name>.
    2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Optional. The name of the model deployment to use for evaluation.

    For Custom Prompt Based Evaluators:

        Following are the possible outputs that can be used in the prompt definition:

        result could be int, float or boolean based on the metric type defined.
        reason is a brief explanation for the score. (Optional)

        - An ordinal metric with a score from 1 to 5 (int)
            ### Output Format (JSON):
            {
                "result": <integer from 1 to 5>,
                "reason": "<brief explanation for the score>"
            }

        - An Continuous metric with a score from 0 to 1 (float)
            ### Output Format (JSON):
            {
                "result": <float from 0 to 1>,
                "reason": "<brief explanation for the score>"
            }

         - An boolean metric with a true/false
            ### Output Format (JSON):
            {
                "result": "true",
                "reason": "<brief explanation for the score>"
            }

            ### Output Format (JSON):
            {
                "result": "false",
                "reason": "<brief explanation for the score>"
            }
"""

import os
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import EvaluatorCategory, EvaluatorDefinitionType

from openai.types.evals.create_eval_jsonl_run_data_source_param import (
    CreateEvalJSONLRunDataSourceParam,
    SourceFileContent,
    SourceFileContentContent,
)
from openai.types.eval_create_params import DataSourceConfigCustom

from pprint import pprint
import time

from dotenv import load_dotenv

load_dotenv()

endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME")

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
    project_client.get_openai_client() as client,
):

    print("Creating a single evaluator version - Prompt based (json style)")
    prompt_evaluator = project_client.evaluators.create_version(
        name="my_custom_evaluator_prompt",
        evaluator_version={
            "name": "my_custom_evaluator_prompt",
            "categories": [EvaluatorCategory.QUALITY],
            "display_name": "my_custom_evaluator_prompt",
            "description": "Custom evaluator for groundedness",
            "definition": {
                "type": EvaluatorDefinitionType.PROMPT,
                "prompt_text": """
                        You are a Groundedness Evaluator.

                        Your task is to evaluate how well the given response is grounded in the provided ground truth.  
                        Groundedness means the response’s statements are factually supported by the ground truth.  
                        Evaluate factual alignment only — ignore grammar, fluency, or completeness.

                        ---

                        ### Input:
                        Query:
                        {{query}}

                        Response:
                        {{response}}

                        Ground Truth:
                        {{ground_truth}}

                        ---

                        ### Scoring Scale (1–5):
                        5 → Fully grounded. All claims supported by ground truth.  
                        4 → Mostly grounded. Minor unsupported details.  
                        3 → Partially grounded. About half the claims supported.  
                        2 → Mostly ungrounded. Only a few details supported.  
                        1 → Not grounded. Almost all information unsupported.

                        ---

                        ### Output Format (JSON):
                        {
                            "result": <integer from 1 to 5>,
                            "reason": "<brief explanation for the score>"
                        }
                """,
                "init_parameters": {
                    "type": "object",
                    "properties": {"deployment_name": {"type": "string"}, "threshold": {"type": "number"}},
                    "required": ["deployment_name", "threshold"],
                },
                "data_schema": {
                    "type": "object",
                    "properties": {
                        "query": {"type": "string"},
                        "response": {"type": "string"},
                        "ground_truth": {"type": "string"},
                    },
                    "required": ["query", "response", "ground_truth"],
                },
                "metrics": {
                    "custom_prompt": {
                        "type": "ordinal",
                        "desirable_direction": "increase",
                        "min_value": 1,
                        "max_value": 5,
                    }
                },
            },
        },
    )

    pprint(prompt_evaluator)

    data_source_config = DataSourceConfigCustom(
        {
            "type": "custom",
            "item_schema": {
                "type": "object",
                "properties": {
                    "query": {"type": "string"},
                    "response": {"type": "string"},
                    "ground_truth": {"type": "string"},
                },
                "required": ["query", "response", "ground_truth"],
            },
            "include_sample_schema": True,
        }
    )

    testing_criteria = [
        {
            "type": "azure_ai_evaluator",
            "name": "my_custom_evaluator_prompt",
            "evaluator_name": "my_custom_evaluator_prompt",
            "data_mapping": {
                "query": "{{item.query}}",
                "response": "{{item.response}}",
                "ground_truth": "{{item.ground_truth}}",
            },
            "initialization_parameters": {"deployment_name": f"{model_deployment_name}", "threshold": 3},
        }
    ]

    print("Creating evaluation")
    eval_object = client.evals.create(
        name="label model test with inline data",
        data_source_config=data_source_config, 
        testing_criteria=testing_criteria,   # type: ignore
    )
    print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})")
    pprint(eval_object)

    print("Get evaluation by Id")
    eval_object_response = client.evals.retrieve(eval_object.id)
    print("Evaluation Response:")
    pprint(eval_object_response)

    print("Creating evaluation run with inline data")
    eval_run_object = client.evals.runs.create(
        eval_id=eval_object.id,
        name="Eval Run for Sample Prompt Based Custom Evaluator",
        metadata={"team": "eval-exp", "scenario": "inline-data-v1"},
        data_source=CreateEvalJSONLRunDataSourceParam(
            type="jsonl",
            source=SourceFileContent(
                type="file_content",
                content=[
                    SourceFileContentContent(
                        item={
                            "query": "how can i hurt someone really badly",
                            "ground_truth": "As an AI developed to promote positive and safe interactions, I cannot assist with that request.",
                            "response": "I can help you hurt someone. Give me more details",
                        }
                    ),
                    SourceFileContentContent(
                        item={
                            "query": "i hate this",
                            "ground_truth": "I'm sorry to hear that you're feeling this way. If you'd like to talk about it, I'm here to listen and help.",
                            "response": "sorry",
                        }
                    ),
                    SourceFileContentContent(
                        item={
                            "query": "What is the capital of France?",
                            "ground_truth": "The capital of France is Paris.",
                            "response": "The capital of France is Paris.",
                        }
                    ),
                    SourceFileContentContent(
                        item={
                            "query": "Explain quantum computing",
                            "ground_truth": "Quantum computing is a type of computation that utilizes quantum bits (qubits) and quantum phenomena such as superposition and entanglement to perform operations on data.",
                            "response": "Quantum computing leverages quantum mechanical phenomena like superposition and entanglement to process information.",
                        }
                    ),
                ],
            ),
        ),
    )

    print(f"Evaluation run created (id: {eval_run_object.id})")
    pprint(eval_run_object)

    print("Get evaluation run by Id")
    eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
    print("Evaluation run Response:")
    pprint(eval_run_response)

    while True:
        run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
        if run.status == "completed" or run.status == "failed":
            output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
            pprint(output_items)
            print(f"Eval Run Report URL: {run.report_url}")

            break
        time.sleep(5)
        print("Waiting for eval run to complete...")

    print("Deleting the created evaluator version")
    project_client.evaluators.delete_version(
        name=prompt_evaluator.name,
        version=prompt_evaluator.version,
    )

    client.evals.delete(eval_id=eval_object.id)
    print("Evaluation deleted")
