watsonxで日本語のRAGを検証

Detached prompt templateの作成

# %%
from dotenv import load_dotenv

load_dotenv(override=True)

# %%
import os
from ibm_aigov_facts_client import AIGovFactsClient

CLOUD_API_KEY = os.environ.get("CLOUD_API_KEY")
PROJECT_ID = os.environ.get("PROJECT_ID")

facts_client = AIGovFactsClient(
    api_key=CLOUD_API_KEY,
    container_id=PROJECT_ID,
    container_type="project",
    disable_tracing=True
)

# %%
from ibm_aigov_facts_client import DetachedPromptTemplate

AZURE_OPENAI_DEPLOYMENT_NAME = os.environ.get("AZURE_OPENAI_DEPLOYMENT_NAME")
AZURE_OPENAI_ENDPOINT = os.environ.get("AZURE_OPENAI_ENDPOINT")

detached_information = DetachedPromptTemplate(
    prompt_id="rag-azure-openai-gpt-35-turbo",
    model_id=AZURE_OPENAI_DEPLOYMENT_NAME,
    model_provider="azure-ml-openai",
    model_name=AZURE_OPENAI_DEPLOYMENT_NAME,
    model_url=AZURE_OPENAI_ENDPOINT,
    prompt_url="prompt_url",
    prompt_additional_info={"model_owner": "Microsoft"}
)

# %%
template = """# 指示:
与えられた文脈をもとに質問に回答してください。

# 文脈:
{context}

# 質問:
{question}

# 回答:
"""

# %%
from ibm_aigov_facts_client import PromptTemplate

prompt_template = PromptTemplate(
    input=template,
    prompt_variables={"context": "", "question": ""},
    input_prefix="",
    output_prefix="",
)

# %%

response = facts_client.assets.create_detached_prompt(
    name="rag-azure-openai-gpt-35-turbo",
    model_id=AZURE_OPENAI_DEPLOYMENT_NAME,
    task_id="retrieval_augmented_generation",
    detached_information=detached_information,
    description="RAG based on Azure OpenAI GPT-3.5-turbo",
    prompt_details=prompt_template)

# %%
response

# %%

Detached prompt templateの設定

# %%
from dotenv import load_dotenv

load_dotenv(override=True)

# %%
import os

SERVICE_INSTANCE_ID = os.environ.get("SERVICE_INSTANCE_ID")
CLOUD_API_KEY = os.environ.get("CLOUD_API_KEY")

# %%
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
from ibm_watson_openscale import *
from ibm_watson_openscale.supporting_classes.enums import *
from ibm_watson_openscale.supporting_classes import *

authenticator = IAMAuthenticator(apikey=CLOUD_API_KEY)
wos_client = APIClient(authenticator=authenticator, service_instance_id=SERVICE_INSTANCE_ID)
data_mart_id = wos_client.service_instance_id
print(data_mart_id)
print(wos_client.version)

# %%
wos_client.data_marts.show()

# %%
language_code = "ja"

supporting_monitors = {
    "generative_ai_quality": {
        "parameters": {
            "min_sample_size": 10,
            "metrics_configuration": {
            }
        }
    }
}

# %%
PROMPT_TEMPLATE_ASSET_ID = os.environ.get("PROMPT_TEMPLATE_ASSET_ID")
PROJECT_ID = os.environ.get("PROJECT_ID")

# %%
response = wos_client.wos.execute_prompt_setup(
    prompt_template_asset_id=PROMPT_TEMPLATE_ASSET_ID,
	label_column="reference_text",
	operational_space_id="development",
	problem_type="retrieval_augmented_generation",
	input_data_type="unstructured_text",
    data_input_locale=[language_code],
    generated_output_locale=[language_code],
	project_id=PROJECT_ID,
    context_fields=["context"],     
    question_field="question",
	supporting_monitors=supporting_monitors,
	background_mode=True)
response.result.to_dict()

# %%
response = wos_client.wos.get_prompt_setup(
    prompt_template_asset_id=PROMPT_TEMPLATE_ASSET_ID,
    project_id=PROJECT_ID)
response.result.to_dict()

# %%
subscription_id = response.result.to_dict()['subscription_id']
subscription_id

# %%
wos_client.monitor_instances.show(target_target_id=subscription_id)

検証データの作成

# %%
import pandas as pd

df = pd.read_csv("RAG-Evaluation-Dataset-JA-generated_text.csv")
df

# %%
df.rename(columns={'target_answer': 'reference_text'}, inplace=True)
df

# %%
df_ = df.loc[df["domain"] == "finance", ["context", "question", "generated_text", "reference_text"]].copy()
df_

# %%
import csv

test_data_path = "test.csv"
df_.to_csv(test_data_path, index=False, quoting=csv.QUOTE_ALL)

評価の実施

# %% Read the MRM monitor instance id
monitor_definition_id = "mrm"
target_target_id = subscription_id
response = wos_client.monitor_instances.list(
    data_mart_id=data_mart_id,
    monitor_definition_id=monitor_definition_id,
    target_target_id=target_target_id,
    project_id=PROJECT_ID)
response.result.to_dict()

# %%
mrm_monitor_instance_id = response.result.to_dict()["monitor_instances"][0]["metadata"]["id"]
mrm_monitor_instance_id
# %%
test_data_set_name = "data"
content_type = "multipart/form-data"

response = wos_client.monitor_instances.mrm.evaluate_risk(
    monitor_instance_id=mrm_monitor_instance_id, 
    test_data_set_name=test_data_set_name, 
    test_data_path=test_data_path,
    content_type=content_type,
    body=None,
    includes_model_output=True,
    project_id=PROJECT_ID,
    background_mode=True)
response.result.to_dict()

# %%
response = wos_client.monitor_instances.mrm.get_risk_evaluation(
    monitor_instance_id=mrm_monitor_instance_id, project_id=PROJECT_ID)
response.result.to_dict()

評価の可視化

# %% 生成AI性能指標の可視化
monitor_definition_id = "generative_ai_quality"
response = wos_client.monitor_instances.list(
    data_mart_id=data_mart_id,
    monitor_definition_id=monitor_definition_id,
    target_target_id=target_target_id,
    project_id=PROJECT_ID)
response.result.to_dict()

# %%
gaiquality_monitor_instance_id = response.result.to_dict()["monitor_instances"][0]["metadata"]["id"]
gaiquality_monitor_instance_id

# %%
wos_client.monitor_instances.show_metrics(monitor_instance_id=gaiquality_monitor_instance_id, project_id=project_id)

# %% Display record level metrics for Generative AI Quality
response = wos_client.data_sets.list(
    target_target_id=subscription_id,
    target_target_type="subscription",
    type="gen_ai_quality_metrics")
response.result.to_dict()

# %%
gaiquality_dataset_id = response.result.to_dict()['data_sets'][0]['metadata']['id']
gaiquality_dataset_id

# %%
wos_client.data_sets.show_records(data_set_id=gaiquality_dataset_id)