②watsonx.ai/.governaceによるRAGの検証

PROJECTの作成＆PROJECT_IDの取得

PROMPT TEMPLATEをPROJECTに登録

detached_prompt_template.py

# %%
# %%
from dotenv import load_dotenv

load_dotenv()

# %%
import os
from ibm_aigov_facts_client import AIGovFactsClient
from ibm_aigov_facts_client import CloudPakforDataConfig

service_url = os.environ.get("CPD_URL")
username = os.environ.get("CPD_USERNAME")
api_key = os.environ.get("CPD_API_KEY")
creds=CloudPakforDataConfig(
    service_url=service_url,
    username=username,
    api_key=api_key
)

project_id = os.environ.get("PROJECT_ID")
facts_client = AIGovFactsClient(
    cloud_pak_for_data_configs=creds,
    container_id=project_id,
    container_type="project",
    disable_tracing=True
)

# %%
from ibm_aigov_facts_client import DetachedPromptTemplate, PromptTemplate

tmp = os.environ.get("AZURE_OPENAI_DEPLOYMENT_NAME")
detached_information = DetachedPromptTemplate(
    prompt_id="detached-aoai-prompt",
    model_id=f"azure/{tmp}",
    model_provider="Azure OpenAI",
    model_name="GPT-3.5-turbo",
    model_url=os.environ.get("AZURE_OPENAI_ENDPOINT"),
    prompt_url="prompt_url",
    prompt_additional_info={"model_owner": "Microsoft", "model_version": "gpt-3.5-turbo-1106"}
)

# %%
template = """# System:
Answer the question based on the given context.

# Context:
{context}

# Question:
{question}

# Answer:
"""

# %%
prompt_variables = {"context": "", "question": ""}
input = template
input_prefix = ""
output_prefix = ""

prompt_template = PromptTemplate(
    input=input,
    prompt_variables=prompt_variables,
    input_prefix=input_prefix,
    output_prefix=output_prefix,
)

# %%
name = "RAG based on Azure OpenAI GPT-3.5-turbo"
description = name
tmp = os.environ.get("AZURE_OPENAI_DEPLOYMENT_NAME")
model_id = f"azure/{tmp}"
task_id = "retrieval_augmented_generation"

response = facts_client.assets.create_detached_prompt(
    model_id=model_id,
    task_id=task_id,
    name=name,
    description=description,
    prompt_details=prompt_template,
    detached_information=detached_information)

# %%
response

# %%

検証データの作成

context、question、reference_output（正解）と生成された回答（generated_output）の4つが必要。

generated_output.py

# %%
from dotenv import load_dotenv
import os

load_dotenv()

# %%
import pandas as pd

filepath = "data.csv"
df = pd.read_csv(filepath_or_buffer=filepath)
df

# %%
from rag import llm, prompt_template

# %%
generated_output = []
for index, row in df.iterrows():
    print(index)
    context = row.context
    question = row.question
    prompt = prompt_template.format(context=context, question=question)

    try:
        response = llm.invoke(input=prompt)
        generated_output.append(response.content)
    except Exception as e:
        generated_output.append("")

# %%
generated_output

# %%
df['generated_output'] = generated_output
df

# %%
import csv

df.iloc[:10].to_csv("test.csv", index=False, quoting=csv.QUOTE_ALL)

# %%

検証

結果