[update]申請履歴のCSVのカラムを柔軟な形にして、動作確認までした。

log_config.json

@@ -39,6 +39,11 @@
             "handlers": ["consoleHandler", "fileHandler"],
             "propagate": false
         },
+        "src.aws.s3": {
+            "level": "DEBUG",
+            "handlers": ["consoleHandler", "fileHandler"],
+            "propagate": false
+        },
         "src.utils": {
             "level": "DEBUG",
             "handlers": ["consoleHandler", "fileHandler"],

main.py

@@ -1,7 +1,7 @@
 import streamlit as st
 
 from src.forms.change_request import change_request_form
-from src.forms.database_check import database_check_form
+from src.forms.change_request_history import change_request_history
 from src.forms.deviation_occurrence_report import deviation_occurrence_report_form
 from src.forms.deviation_report import deviation_report_form
 
@@ -9,7 +9,7 @@ pages = {
     "変更申請書": change_request_form,
     "逸脱発生報告書": deviation_occurrence_report_form,
     "逸脱報告書": deviation_report_form,
-    "申請履歴": database_check_form,
+    "変更申請履歴": change_request_history,
 }
 
 selected_page = st.sidebar.selectbox(
@@ -22,5 +22,5 @@ match selected_page:
         pages["逸脱発生報告書"]()
     case "逸脱報告書":
         pages["逸脱報告書"]()
-    case "申請履歴":
-        pages["申請履歴"]()
+    case "変更申請履歴":
+        pages["変更申請履歴"]()

requirements.txt

@@ -12,3 +12,4 @@ pandas == 2.2.2
 boto3 == 1.34.131
 pandas-stubs == 2.2.2.240603
 boto3-stubs == 1.34.149
+botocore == 1.34.155

src/aws/s3.py

@@ -2,22 +2,62 @@ from io import StringIO
 
 import boto3
 import pandas as pd
+from botocore.client import BaseClient
+from botocore.exceptions import NoCredentialsError, PartialCredentialsError
 
-s3 = boto3.client("s3")
+from src.utils import setup_logger
 
+logger = setup_logger(__name__)
 
-# Download the CSV file from S3
-def download_csv_panda_dataframe_from_s3(
-    bucket_name: str, file_name: str
+trial_bucket_name = "pharma-doc-trials"
+change_request_history_file_path_in_trial_bucket = "bankyo/change_request_history.csv"
+csv_extension = ".csv"
+err_msg_file_is_not_csv = "指定したファイルはcsvではありません。"
+
+
+def get_client() -> BaseClient:
+    return boto3.client("s3")
+
+
+def get_csv_as_pd_dataframe_from_s3(
+    client: BaseClient, bucket_name: str, file_name: str
 ) -> pd.DataFrame:
-    obj = s3.get_object(Bucket=bucket_name, Key=file_name)
-    data = pd.read_csv(obj["Body"])
-    return data
+    if not file_name.lower().endswith(csv_extension):
+        raise ValueError(err_msg_file_is_not_csv)
+    try:
+        obj = client.get_object(Bucket=bucket_name, Key=file_name)
+        data = pd.read_csv(obj["Body"])
+        return data
+    except client.exceptions.NoSuchKey as e:
+        logger.info(f'No such key: {e.response["Error"]["Key"]}')
+        raise e
+    except NoCredentialsError as e:
+        logger.info("Error: No AWS credentials found.")
+        raise e
+    except PartialCredentialsError as e:
+        logger.info("Error: Incomplete AWS credentials found.")
+        raise e
+    except client.exceptions.ClientError as e:
+        logger.info(f"Unexpected error: {e}")
+        raise e
 
 
-def save_csv_panda_dataframe_to_s3(
-    data: pd.DataFrame, bucket_name: str, file_name: str
+def save_pd_dataframe_as_csv_to_s3(
+    client: BaseClient, data: pd.DataFrame, bucket_name: str, file_name: str
 ) -> None:
+    if not file_name.lower().endswith(csv_extension):
+        raise ValueError(err_msg_file_is_not_csv)
     csv_buffer = StringIO()
     data.to_csv(csv_buffer, index=False)
-    s3.put_object(Bucket=bucket_name, Key=file_name, Body=csv_buffer.getvalue())
+    try:
+        client.put_object(Bucket=bucket_name, Key=file_name, Body=csv_buffer.getvalue())
+        logger.info(f"Successfully saved {file_name} to {bucket_name}.")
+    except NoCredentialsError as e:
+        logger.info("Error: No AWS credentials found.")
+        raise e
+    except PartialCredentialsError as e:
+        logger.info("Error: Incomplete AWS credentials found.")
+        raise e
+    except client.exceptions.ClientError as e:
+        logger.info(f"Unexpected error: {e}")
+        raise e

src/database.py

@@ -1,64 +0,0 @@
-from datetime import datetime
-from typing import List
-
-import pandas as pd
-import streamlit as st
-
-from src.aws.s3 import (
-    download_csv_panda_dataframe_from_s3,
-    save_csv_panda_dataframe_to_s3,
-)
-
-bucket_name = "pharma-doc-trials"
-path_name = "bankyo/change_request_history.csv"
-
-
-def write_csv(
-    applicant_name: str,
-    change_type: List[str],
-    category: List[str],
-    subcategory: List[str],
-    questions: List[str],
-    input: List[str],
-    output_change_content: str,
-    output_change_reason: str,
-    output_effect: str,
-) -> None:
-    all_types = []
-    for i in range(len(change_type)):
-        all_type = [i + 1, change_type[i], category[i], subcategory[i]]
-        all_types.append(all_type)
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    data = download_csv_panda_dataframe_from_s3(bucket_name, path_name)
-    new_row = [
-        timestamp,
-        applicant_name,
-        all_types,
-        output_change_content,
-        output_change_reason,
-        output_effect,
-    ]
-    for change_serial_number in range(len(questions)):
-        new_row.append(questions[change_serial_number])
-        new_row.append(input[change_serial_number])
-
-    max_length = max(len(data.columns), len(new_row))
-    number_of_qeustions = len(questions)
-    column_existed_number_of_question = (len(data.columns) - 6) // 2
-    if len(data.columns) < max_length:
-        for i in range(column_existed_number_of_question, number_of_qeustions):
-            data[f"質問{i+1}"] = None
-            data[f"回答{i+1}"] = None
-    new_columns = data.columns.tolist()
-    while len(new_row) < len(new_columns):
-        new_row.append("")
-    new_data = pd.DataFrame([new_row], columns=new_columns)
-    data = pd.concat([data, new_data], ignore_index=True)
-    save_csv_panda_dataframe_to_s3(data, bucket_name, path_name)
-
-
-def display_csv(file_path: str) -> None:
-    data = download_csv_panda_dataframe_from_s3(bucket_name, path_name)
-    data.to_csv(file_path, index=False)
-    st.write("**データベース**")
-    st.dataframe(data)

src/forms/change_request.py

@@ -1,13 +1,22 @@
 import json
 import os
 from datetime import date, datetime
+from typing import Any, List
+from zoneinfo import ZoneInfo
 
+import pandas as pd
 import streamlit as st
 from dotenv import load_dotenv
 from openai import OpenAI
 from pinecone import Pinecone
 
-from src.database import display_csv, write_csv
+from src.aws.s3 import change_request_history_file_path_in_trial_bucket
+from src.aws.s3 import get_client as get_s3_client
+from src.aws.s3 import (
+    get_csv_as_pd_dataframe_from_s3,
+    save_pd_dataframe_as_csv_to_s3,
+    trial_bucket_name,
+)
 from src.prompts.change_request import (
     system_template_review,
     systemprompt_template_1,
@@ -141,6 +150,11 @@ SUBCATEGORIES = {
     ): ["", "液", "クリーム", "チューブ"],
 }
 
+change_request_history_file_path = "data/change_request_history.csv"
+maximum_change_number: int = 10
+start_from_one: int = 1
+upper_bound_correction: int = 1
+
 logger = setup_logger(__name__)
 
 
@@ -165,7 +179,6 @@ def change_request_form() -> None:
         if state not in st.session_state:
             st.session_state[state] = False
 
-    upper_bound_correction: int = 1
     with st.form("common_question_form"):
         col1, col2 = st.columns(2)
         with col1:
@@ -179,7 +192,6 @@ def change_request_form() -> None:
 
         col2_1, col2_2 = st.columns(2)
         minimum_change_number: int = 1
-        maximum_change_number: int = 10
         with col2_1:
             st.selectbox(
                 "変更の件数",
@@ -207,7 +219,6 @@ def change_request_form() -> None:
 
     change_number = int(st.session_state["change_num"])
 
-    start_from_one: int = 1
     for change_serial_number in range(
         start_from_one, change_number + upper_bound_correction
     ):
@@ -280,10 +291,10 @@ def change_request_form() -> None:
                 st.session_state[f"category_{change_serial_number}"],
                 st.session_state[f"subcategory_{change_serial_number}"],
             )
-            for change_serial_number in range(len(individual_questions)):
+            for i in range(len(individual_questions)):
                 next_question_number = len(question_about_this_change) + 1
                 question_about_this_change.append(
-                    f"Q{change_serial_number}-{next_question_number} : {individual_questions[change_serial_number]}"
+                    f"Q{change_serial_number}-{next_question_number} : {individual_questions[i]}"
                 )
             question_about_this_change.append(
                 f"Q{change_serial_number}-"
@@ -359,11 +370,12 @@ def change_request_form() -> None:
                 )
                 user_prompt = ""
                 # db用
-                user_input = []
+                answers_of_all_changes = []
 
                 for change_serial_number in range(
                     start_from_one, change_number + upper_bound_correction
                 ):
+                    answer_about_this_change = []
                     for question_index in range(
                         len(questions_of_all_changes[change_serial_number - 1])
                     ):
@@ -383,7 +395,8 @@ def change_request_form() -> None:
                         # question は「Q1-12...」などの形式
                         answer_id = question.split(" : ")[0].replace("Q", "A")
                         user_prompt += f"{answer_id} :  {answer}\n"
-                        user_input.append(f"{answer_id} :  {answer}\n")
+                        answer_about_this_change.append(f"{answer_id} :  {answer}")
+                    answers_of_all_changes.append(answer_about_this_change)
 
                 system_prompt += systemprompt_template_3
 
@@ -430,31 +443,33 @@ def change_request_form() -> None:
 
                 output_change_application(st.session_state["contents"])
 
-                type = []
-                category = []
-                subcategory = []
+                types = []
+                categories = []
+                subcategories = []
 
                 for change_serial_number in range(
                     start_from_one, change_number + upper_bound_correction
                 ):
-                    type.append(st.session_state[f"type_{change_serial_number}"])
-                    category.append(
+                    types.append(st.session_state[f"type_{change_serial_number}"])
+                    categories.append(
                         st.session_state[f"category_{change_serial_number}"]
                     )
-                    subcategory.append(
+                    subcategories.append(
                         st.session_state[f"subcategory_{change_serial_number}"]
                     )
 
-                write_csv(
+                save_change_request_history(
                     st.session_state["applicant_name"],
-                    type,
-                    category,
-                    subcategory,
-                    question_about_this_change,
-                    user_input,
+                    supplementary_information,
+                    attached_files,
                     str(response_data["変更内容"]),
                     str(response_data["変更理由"]),
                     str(response_data["品質への影響"]),
+                    types,
+                    categories,
+                    subcategories,
+                    questions_of_all_changes,
+                    answers_of_all_changes,
                 )
                 st.session_state["document_generated"] = True
 
@@ -484,7 +499,17 @@ def change_request_form() -> None:
         st.markdown(f'**備考** : {st.session_state["contents"]["備考"]}')
 
         review_button = st.button("レビューする")
-        display_csv("data/change_request_history.csv")
+
+        bucket_name = "pharma-doc-trials"
+
+        s3_client = get_s3_client()
+        data = get_csv_as_pd_dataframe_from_s3(
+            s3_client, bucket_name, change_request_history_file_path_in_trial_bucket
+        )
+        data.to_csv(change_request_history_file_path, index=False)
+        st.write("**変更申請履歴**")
+        st.dataframe(data)
+
         if review_button:
             query_content = st.session_state["contents"]["変更内容"]
             query_reason = st.session_state["contents"]["変更理由"]
@@ -553,3 +578,125 @@ def change_request_form() -> None:
             )
             logger.info("レビュー結果: " + review)
             st.markdown(f"**レビュー結果** : {review}")
+
+
+def save_change_request_history(
+    applicant_name: str,
+    supplementary_information: str,
+    attached_files: str,
+    output_change_content: str,
+    output_change_reason: str,
+    output_effect: str,
+    change_type: List[str],
+    category: List[str],
+    subcategory: List[str],
+    questions: List[List[str]],
+    answers: List[List[str]],
+) -> None:
+    s3_client = get_s3_client()
+    try:
+        df = get_csv_as_pd_dataframe_from_s3(
+            s3_client,
+            trial_bucket_name,
+            change_request_history_file_path_in_trial_bucket,
+        )
+    except s3_client.exceptions.NoSuchKey:
+        df = pd.DataFrame()
+
+    new_common_headers = generate_headers()
+    question_answer_headers = generate_question_answer_headers(questions)
+    new_common_headers.extend(question_answer_headers)
+
+    existing_headers = df.columns.tolist()
+    new_columns = [
+        header for header in new_common_headers if header not in existing_headers
+    ]
+
+    for column in new_columns:
+        df[column] = ""
+
+    new_row_data = create_new_row_data(
+        applicant_name,
+        supplementary_information,
+        attached_files,
+        output_change_content,
+        output_change_reason,
+        output_effect,
+        change_type,
+        category,
+        subcategory,
+        questions,
+        answers,
+    )
+
+    new_row = pd.DataFrame([new_row_data])
+    df = pd.concat([df, new_row], ignore_index=True)
+
+    save_pd_dataframe_as_csv_to_s3(
+        s3_client,
+        df,
+        trial_bucket_name,
+        change_request_history_file_path_in_trial_bucket,
+    )
+
+
+def generate_headers() -> List[str]:
+    new_headers = [
+        "申請者",
+        "備考",
+        "添付資料",
+        "作成日時",
+        "変更内容",
+        "変更理由",
+        "品質への影響",
+    ]
+    for i in range(start_from_one, maximum_change_number + upper_bound_correction):
+        new_headers.extend(
+            [f"変更の種類({i}件目)", f"大分類({i}件目)", f"小分類({i}件目)"]
+        )
+    return new_headers
+
+
+def generate_question_answer_headers(questions: List[List[str]]) -> List[str]:
+    headers = []
+    for i, question_set in enumerate(questions):
+        for j, _ in enumerate(question_set):
+            headers.append(f"質問{i+1}-{j+1}")
+            headers.append(f"回答{i+1}-{j+1}")
+    return headers
+
+
+def create_new_row_data(
+    applicant_name: str,
+    supplementary_information: str,
+    attached_files: str,
+    output_change_content: str,
+    output_change_reason: str,
+    output_effect: str,
+    change_type: List[str],
+    category: List[str],
+    subcategory: List[str],
+    questions: List[List[str]],
+    answers: List[List[str]],
+) -> dict[str, Any]:
+    now = datetime.now(ZoneInfo("Asia/Tokyo"))
+    data = {
+        "申請者": applicant_name,
+        "備考": supplementary_information,
+        "添付資料": attached_files,
+        "作成日時": now,
+        "変更内容": output_change_content,
+        "変更理由": output_change_reason,
+        "品質への影響": output_effect,
+    }
+    for i in range(maximum_change_number):
+        data[f"変更の種類({i+1}件目)"] = change_type[i] if i < len(change_type) else ""
+        data[f"大分類({i+1}件目)"] = category[i] if i < len(category) else ""
+        data[f"小分類({i+1}件目)"] = subcategory[i] if i < len(subcategory) else ""
+    for i, question_set in enumerate(questions):
+        for j, question in enumerate(question_set):
+            data[f"質問{i+1}-{j+1}"] = question
+            data[f"回答{i+1}-{j+1}"] = (
+                answers[i][j] if i < len(answers) and j < len(answers[i]) else ""
+            )
+    return data

src/forms/change_request_history.py

@@ -0,0 +1,46 @@
+import os
+
+import streamlit as st
+
+from src.aws.s3 import (
+    change_request_history_file_path_in_trial_bucket,
+    get_client,
+    get_csv_as_pd_dataframe_from_s3,
+    trial_bucket_name,
+)
+from src.forms.change_request import change_request_history_file_path
+
+
+def change_request_history() -> None:
+    s3_client = get_client()
+    try:
+        data = get_csv_as_pd_dataframe_from_s3(
+            s3_client,
+            trial_bucket_name,
+            change_request_history_file_path_in_trial_bucket,
+        )
+    except s3_client.exceptions.NoSuchKey:
+        st.write("変更履歴がありません。")
+        return
+    except Exception:
+        st.error(
+            "システムの内部エラーにより、申請履歴の表示ができません。システム管理者に問い合わせて下さい。"
+        )
+        return
+    data.to_csv(change_request_history_file_path, index=False)
+    st.write("**変更申請履歴**")
+    st.dataframe(data)
+    file_exists = os.path.isfile(change_request_history_file_path)
+    if file_exists:
+        with open(change_request_history_file_path, "rb") as file:
+            st.download_button(
+                label="ダウンロード",
+                data=file,
+                file_name="change_request_history.csv",
+                mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            )
+    else:
+        st.error(
+            "システムの内部エラーにより、申請履歴の表示ができません。システム管理者に問い合わせて下さい。"
+        )
+        return

src/forms/database_check.py

@@ -1,19 +0,0 @@
-import os
-
-import streamlit as st
-
-from src.database import display_csv
-
-
-def database_check_form() -> None:
-    file_path = "data/change_request_history.csv"
-    display_csv(file_path)
-    file_exists = os.path.isfile(file_path)
-    if file_exists:
-        with open(file_path, "rb") as file:
-            st.download_button(
-                label="ダウンロード",
-                data=file,
-                file_name="change_request_history.csv",
-                mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            )