app.py

import re
import os
import json
import time
import fitz
import pymongo
import difflib
import secrets
import openai
import string
import logging
import requests
import streamlit as st
from pandas import DataFrame
from openai import AzureOpenAI
from bson.json_util import dumps
from difflib import SequenceMatcher
from tempfile import NamedTemporaryFile
from typing import Tuple, Dict, List, Union
from diff_match_patch import diff_match_patch

import base64
import pandas as pd


# Your existing functions go here

def clean_string(text: str, stem: str = "None") -> str:
    """
    Clean the input text by removing punctuation, numbers, and optionally stemming or lemmatizing the words.

    Args:
        text (str): The input text to be cleaned.
        stem (str, optional): The stemming method to be used. Options are "None" (default), "Stem", "Lem", or "Spacy".

    Returns:
        str: The cleaned text.

    Raises:
        None
    """
    try:
        final_string = ""
        # Replace any characters with nothing or a space
        text = re.sub(r'\n', ' ', text)
        text = re.sub(r' +', ' ', text)
        text = re.sub(r'[^\x00-\x7f]',r'', text)
        # Remove punctuation
        translator = str.maketrans('', '', string.punctuation)
        text = text.translate(translator)
        # Remove numbers
        text_filtered = [re.sub(r'\w*\d\w*\s+', '', w) for w in text]
        text_filtered = [re.sub('[0-9]', '', w) for w in text_filtered]
        # Stem or Lemmatize
        if stem == 'Stem':
            stemmer = PorterStemmer()
            text_stemmed = [stemmer.stem(y) for y in text_filtered]
        elif stem == 'Lem':
            lem = WordNetLemmatizer()
            text_stemmed = [lem.lemmatize(y) for y in text_filtered]
        elif stem == 'Spacy':
            text_filtered = nlp(' '.join(text_filtered))
            text_stemmed = [y.lemma_ for y in text_filtered]
        else:
            text_stemmed = text_filtered

        final_string = ''.join(text_stemmed)
    except Exception as e:
        # Handle the exception here
        logging.error(f"An error occurred in clean_string: {e}")
        return None

    return final_string

def extract_text_without_header_footer(page: fitz.Page, page_height: float, header_height: float, footer_height: float) -> str:
    """
    Extract the text from a page of a PDF document, excluding the header and footer.

    Args:
        page (fitz.Page): The page object from the PyMuPDF library.
        page_height (float): The height of the page.
        header_height (float): The height of the header to be excluded.
        footer_height (float): The height of the footer to be excluded.

    Returns:
        str: The extracted text.

    Raises:
        None
    """
    try:
        exclude_top = header_height
        exclude_bottom = page_height - footer_height
        text = ""
        for text_block in page.get_text_blocks():
            bbox = fitz.Rect(text_block[:4])  # Get bounding box of text block
            if bbox.y0 >= exclude_top and bbox.y1 <= exclude_bottom:
                text += text_block[4] + "\n"  # Add text content to result
        return text
    except Exception as e:
        # Handle the exception here
        logging.error(f"An error occurred in extract_text_without_header_footer: {e}")
        return ""

def rect_to_dict(rect: fitz.Rect) -> Dict[str, float]:
    """
    Convert a PyMuPDF Rect object to a dictionary.

    Args:
        rect (fitz.Rect): The Rect object from the PyMuPDF library.

    Returns:
        Dict[str, float]: The dictionary representation of the Rect object.

    Raises:
        None
    """
    return {
        "x1": rect[0],
        "y1": rect[1],
        "x2": rect[2],
        "y2": rect[3]
    }

def extract_sections(pdf_path: str, header_height: float, footer_height: float) -> Dict[str, Dict[str, List[str]]]:
    """
    Extract sections from a PDF document, excluding the header and footer.

    Args:
        pdf_path (str): The path to the PDF document.
        header_height (float): The height of the header to be excluded.
        footer_height (float): The height of the footer to be excluded.

    Returns:
        Dict[str, Dict[str, List[str]]]: A dictionary containing the extracted sections.

    Raises:
        None
    """
    try:
        doc = fitz.open(pdf_path)
        sections = {}
        current_section = None
        for page_num in range(2, len(doc)):
            page = doc.load_page(page_num)
            page_height = page.rect.height
            text = extract_text_without_header_footer(page, page_height, header_height, footer_height)
            lines = text.split('\n')
            print("start")
            for line in lines:
                match = re.match(r'^(?:\d+\.\d*(?:\.\d+)*)|AT&T ALLIANCE PROGRAM AGREEMENT', line.strip())
                if match:
                    print("line:",line)
                    section_num = match.group()
                    if section_num != current_section:
                        current_section = section_num
                        sections[current_section] = {'page_num': str(page_num + 1), 'coords': [], 'content': []}

                if current_section:
                    sections[current_section]['content'].append(line)
                    bbox = page.search_for(line)
                    if bbox:
                        for cord in bbox:
                            sections[current_section]['coords'].append(rect_to_dict(cord))
        print("section-keys:",sections.keys())
        for key in sections.keys():
            if 'content' in sections[key]:
                sections[key]['content'] = (' '.join(sections[key]['content']))[len(key):].lstrip()
        print("Content:",sections)
        return sections
    except Exception as e:
        # Handle the exception here
        logging.error(f"An error occurred in extract_sections: {e}")
        return {}

def check_identify_changes(template_text: str, contract_text: str) -> str:
    """
    Compare the template text with the contract text and identify the changes made.

    Args:
        template_text (str): The template text.
        contract_text (str): The contract text.

    Returns:
        str: A sentence prompt describing the changes made.

    Raises:
        None
    """
    try:
      dmp = diff_match_patch()
      diff = dmp.diff_main(template_text, contract_text)
      dmp.diff_cleanupSemantic(diff)

      if  all([True if each[0] == 0 or each[1] == ' ' else False for each in diff]):
        return('no_change')
      else:
        deleted = []
        added = []
        for each in diff:
          if each[0] == -1:
            deleted.append(each[1])
          elif each[0] == 1:
            added.append(each[1])
        sentence_prompt = f"""
        template text = {template_text}
        contract_text = {contract_text}
        deleted from template --- {deleted}
        added to the actual contract -- {added}
        """
        return(sentence_prompt)
    except Exception as e:
      # Handle the exception here
      logging.error(f"An error occurred in check_identify_changes: {e}")
      return ""



def open_ai(prompt: str) -> str:
    """
    Generate AI response using OpenAI GPT-4 model.

    Args:
        prompt (str): The prompt for the AI model.

    Returns:
        str: The generated AI response.

    Raises:
        None
    """
    stat = 0
    while stat == 0:
        try:
            client = AzureOpenAI(api_key=os.getenv("AZURE_API_KEY"),
                     api_version="2023-07-01-preview",
                     azure_endpoint="https://azureadople.openai.azure.com/")
            conversation = []
            conversation.append({"role": "user", "content": prompt})
            response = client.chat.completions.create(
                model="GPT-3",
                messages=conversation,
                temperature=0,
                max_tokens=3000,
                stop=None
            )
            stat = 1
        except openai.RateLimitError as e:
            logging.error(f"Rate limit error occurred: {e}")
            stat = 0
            time.sleep(60)
        except Exception as e:
            logging.error(f"An error occurred in open_ai: {e}")
            stat = 0

    output = response.choices[0].message.content
    return output

prompt = f""""
As an attorney representing AT&T, your task is to compare the template text and the contract text provided and identify any changes that may impact the agreement or AT&T. Specifically, you need to
analyze the legal implications and considerations of the word 'subcontractors' when it appears in the changed text. Instead of focusing solely on the addition or removal of the letters 'or,' provide a comprehensive analysis based on the complete word 'subcontractors.' Classify the changes as either "minor_change" or "major_change."


Please provide the changed text alone as a separate paragraph under the "Changed:" subheading, and the analysis of the changes as a separate paragraph under the "Analysis:" subheading and at the end add ~!~ and classification like ~!~minor_change or ~!~major_change.


EXAMPLES -
1.
Contract text-
Verify identification credentials including Social Security number, driver’s license, educational credentials, employment history, home address and citizenship indicia;
In connection with providing Access to a Customer’s facilities or systems, comply with any additional investigation or screening requirements required by such Customer as communicated in advance by AT&T;   When (i) the Customer or end user is a federal, state or local government entity, test for use of illicit drugs including opiates, cocaine, cannabinoids, amphetamines, and phencyclidine.
Template text-  Verify identification credentials including Social Security number, driver’s license, educational credentials, employment history, home address and citizenship indicia.
In connection with providing Access to a Customer’s facilities or systems, comply with any additional investigation or screening requirements required by such Customer as communicated in advance by AT&T.  Test for use of illicit drugs including opiates, cocaine, cannabinoids, amphetamines, and phencyclidine

Differences Observed -

Changed: "When (i) the Customer or end user is a federal, state or local government entity"

analysis:
A new sentence is added "When (i) the Customer or end user is a federal, state or local government entity" in the template text. This clause specifies a condition under which the drug testing requirement applies. In the template text, this condition is excluded, meaning that the drug testing requirement would apply regardless of whether the customer or end user is a government entity. This could potentially change the scope and applicability of the drug testing provision~!~major_change






2.
Contract text-
Perform a criminal background check to determine, in the counties, states, and federal court districts where Candidate has lived, worked, or attended school in the previous ten years, whether Candidate has been: (1) convicted of any felony; (2) convicted of a misdemeanor involving violence, theft, computer crimes, fraud, financial crimes, drug distribution, unlawful possession or use of a dangerous weapon, or sexual misconduct; or, (3) listed on any government registry as a sex offender (together, “Conviction”); and In connection with providing Access to a Customer’s facilities or systems, comply with any additional investigation or screening requirements required by such Customer as communicated in advance by AT&T.
Template text-
Perform a criminal background check to determine, in the counties, states, and federal court districts where Candidate has lived, worked, or attended school in the previous ten years, whether Candidate has been: (1) convicted of any felony; (2) convicted of a misdemeanor involving violence, theft, computer crimes, fraud, financial crimes, drug distribution, unlawful possession or use of a dangerous weapon, or sexual misconduct; or, (3) listed on any government registry as a sex offender (together, “Conviction”). In connection with providing Access to a Customer’s facilities or systems, comply with any additional investigation or screening requirements required by such Customer as communicated in advance by AT&T.


Differences Observed -

Changed: "and"

Analysis:
The contract text includes the word "and" which isn't in the template. This could potentially change the interpretation of the agreement, as it could be read as each clause being a separate requirement, rather than a list of requirements~!~major_change

3.
Contract text-
SP may not assign, delegate, or otherwise transfer its rights or obligations under this Agreement, voluntarily or involuntarily, without the prior written consent of AT&T except that
SP may delegate certain obligations hereunder to its Subcontractors as contemplated by this Agreement.
Any attempted assignment, delegation or transfer not consented to in writing will be void. Notwithstanding the foregoing, with notice to the other Party, either Party may assign this Agreement, in whole or in part, to any Affiliate, successor-in-interest or wi t h o u t securing the consent of the other Party. Any assignment of money will be void if (i) the assignor fails to give the non-assigning Party at least thirty (30) days prior written notice, or (ii) the assignment purports to impose upon the non-assigning Party additional costs or obligations in addition to the payment of such money, or (iii) the assignment purports to preclude AT&T from dealing solely and directly with SP in all matters pertaining to this Agreement. This Agreement binds and benefits both Parties and their permitted successors and assigns.

Template text-
SP may not assign, delegate, or otherwise transfer its rights or obligations under this Agreement, voluntarily or involuntarily, without the prior written consent of AT&T except that SP may delegate certain obligations hereunder to its Subcontracts as contemplated by this Agreement. Any attempted assignment, delegation or transfer not consented to in writing will be void. Notwithstanding the foregoing, with notice to the other Party, either Party may assign this Agreement, in whole or in part, to any Affiliate, successor-in-interest or w i t h o u t securing the consent of the other Party. Any assignment of money will be void if (i) the assignor fails to give the non-assigning Party at least thirty (30) days prior written notice, or (ii) the assignment purports to impose upon the non-assigning Party additional costs or obligations in addition to the payment of such money, or (iii) the assignment purports to preclude AT&T from dealing solely and directly with SP in all matters pertaining to this Agreement. This Agreement binds and benefits both Parties and their permitted successors and assigns.

Differences Observed -
Changed: "Subcontractors"

Analysis: The contract text includes the word "Subcontractors" instead of "Subcontracts" as in the template. This change could potentially alter the meaning of the agreement as it specifies a different entity (Subcontractors vs Subcontracts). However, given the context, it seems likely that this is a typographical error in the template text, and the intended meaning remains the same.~!~minor_change


4.
Contract text-
This AGREEMENT is entered into between AT&T Corp., a New York corporation, which sometimes does
business as ACC Business, (“AT&T”) and 3OS Technologies, a NJ LLC (“Solution Provider” or “SP”).
AT&T and Solution Provider may be referred to collectively as the “Parties” or individually as a “Party”.

Template text-
ALLIANCE PROGRAM AGREEMENT This AGREEMENT is entered into between AT&T Corp., a New York corporation, which sometimes does business as ACC Business, (“AT&T”) and ________________________, a __________ corporation (“Solution Provider” or “SP”). AT&T and Solution Provider may be referred to collectively as the “Parties” or individually as a “Party.”

Differences Observed -
Changed:"3OS Technologies, a NJ LLC"

Analysis: " The contract text includes the specific name of the corporation "3OS Technologies, a NJ LLC" instead of a placeholder as in the template. This is a minor change as it is expected that the specific name of the corporation would be filled in the actual contract. "~!~minor_change





"""

def compare_strings(string1: str, string2: str) -> float:
    """
    Compare two strings and return their similarity ratio.

    Args:
        string1 (str): The first string.
        string2 (str): The second string.

    Returns:
        float: The similarity ratio between the two strings.

    Raises:
        None
    """
    matcher = SequenceMatcher(None, string1, string2)
    similarity_ratio = matcher.ratio()
    return similarity_ratio

def get_main_section(section_number: str) -> str:
    """
    Get the main section number from a given section number.

    Args:
        section_number (str): The section number.

    Returns:
        str: The main section number.

    Raises:
        None+
    """
    parts = section_number.split('.')
    if len(parts) > 1:
        parts.pop()

    return ".".join(parts)

def increment_section(section_number: str) -> str:
    """
    Increment a section number by one.

    Args:
        section_number (str): The section number.

    Returns:
        str: The incremented section number.

    Raises:
        None
    """
    parts = section_number.split('.')
    last_part = '0' if not parts[-1] else parts[-1]  # Get the last part of the section number
    # Check if the last part is numeric
    if last_part.isdigit():
        # Convert the last part to an integer and increment it by one
        incremented_last_part = str(int(last_part) + 1)
        # Replace the last part in the parts list
        parts[-1] = incremented_last_part
    else:
        # If the last part is not numeric, return the original section number
        return section_number

    # Join the parts back together with '.'
    incremented_section = '.'.join(parts)
    return incremented_section

def count_subsections(dictionary: Dict[str, Union[str, Dict]]) -> Dict[str, int]:
    """
    Count the number of subsections in a dictionary.

    Args:
        dictionary (Dict[str, Union[str, Dict]]): The dictionary containing the subsections.

    Returns:
        Dict[str, int]: A dictionary with the main section numbers as keys and the count of subsections as values.

    Raises:
        None
    """
    section_counts = {}
    for key in dictionary:
        # Split the key by '.' and take the first part as the section number
        section_number = get_main_section(key)

        # Check if the section number is already in the section_counts dictionary
        if section_number in section_counts:
            # Increment the count for the section
            section_counts[section_number] += 1
        else:
            # Initialize the count for the section
            section_counts[section_number] = 1

    return section_counts

def split_page_section(page_no: str, sec_no: str) -> Tuple[str, Tuple[int]]:
    """
    Split the page number and section number into separate components.

    Args:
        page_no (str): The page number.
        sec_no (str): The section number.

    Returns:
        Tuple[str, Tuple[int]]: A tuple containing the page number and section number.

    Raises:
        None
    """
    try:
        page_number = page_no
        section_numbers = sec_no
        section_numbers = [int(num) if num and num.isdigit() else 0 for num in section_numbers]
        section_number = tuple(section_numbers)
        return (page_number, section_number)
    except Exception as e:
        logging.error(f"An error occurred in split_page_section: {e}")
        return ("", ())

def process_comparisons(result_agreement: Dict[str, Dict], result_template: Dict[str, Dict]) -> Tuple[List[Dict], List[Dict], List[Dict]]:
    """
    Process the comparisons between the agreement and template texts.

    Args:
        result_agreement (Dict[str, Dict]): The agreement text.
        result_template (Dict[str, Dict]): The template text.

    Returns:
        Tuple[List[Dict], List[Dict], List[Dict]]: A tuple containing the changes list, actual list, and changes_ui list.

    Raises:
        None
    """
    # Initialize a flag to track if there are any changes
    changes = {}
    changes_ui = {}
    actual = {}
    compared_sections = []
    actual_list = []
    changes_list = []
    changes_ui_list = []
    sections_added = False
    current_section = 0
    prev_section = 0

    # Iterate through the keys and values of the dictionaries
    for key in result_template:
        current_section = get_main_section(key)
        if sections_added and current_section == prev_section:
            contract_key = increment_section(current_section)
        else:
            contract_key = key

        if prev_section != current_section:
            sections_added = False

        if contract_key in result_agreement:
            if count_subsections(result_agreement)[current_section] != count_subsections(result_template)[current_section]:
                if compare_strings(result_template[key]['content'], result_agreement[contract_key]['content']) < .1:
                    # Actual JSON
                    actual = {}
                    actual["actual"] = ""
                    actual["page_number"] = result_template[key]['page_num']
                    actual["section_number"] = key
                    actual["actual_coords"] = ""
                    actual_list.append(actual)
                    # Changes JSON
                    changes = {}
                    changes["changed"] = result_agreement[contract_key]['content'] + ' Analysis:New clause added' + '~!~addition'
                    changes["changed_page_number"] = result_agreement[contract_key]['page_num']
                    changes["changed_section_number"] = key
                    changes["changed_coords"] = result_agreement[contract_key]['coords']
                    changes_list.append(changes)
                    # changes_ui JSON
                    changes_ui = {}
                    changes_ui["changed"] = result_agreement[contract_key]['content'] + ' Analysis:New clause added' + '~!~addition'
                    changes_ui["changed_page_number"] = result_agreement[contract_key]['page_num']
                    changes_ui["changed_section_number"] = key
                    changes_ui["changed_coords"] = result_agreement[contract_key]['coords']
                    changes_ui_list.append(changes_ui)

                    contract_key = increment_section(contract_key)
                    sections_added = True

            compared_sections.append(contract_key)

            if contract_key in result_agreement:
                try:
                    res = check_identify_changes(clean_string(result_template[key]['content']), clean_string(result_agreement[contract_key]['content']))
                    if res != 'no_change':
                        final_prompt = prompt + res
                        llm_res = open_ai(final_prompt)
                        # Actual JSON
                        actual = {}
                        actual["actual"] = result_template[key]['content']
                        actual["page_number"] = result_template[key]['page_num']
                        actual["section_number"] = key
                        actual["actual_coords"] = result_template[key]['coords']
                        actual_list.append(actual)
                        # Changes JSON
                        changes = {}
                        changes["changed"] = llm_res
                        changes["changed_page_number"] = result_agreement[contract_key]['page_num']
                        changes["changed_section_number"] = key
                        changes["changed_coords"] = result_agreement[contract_key]['coords']
                        changes_list.append(changes)
                        # changes_ui JSON
                        changes_ui = {}
                        changes_ui["changed"] = llm_res
                        changes_ui["changed_page_number"] = result_agreement[contract_key]['page_num']
                        changes_ui["changed_section_number"] = key
                        changes_ui["changed_coords"] = result_agreement[contract_key]['coords']
                        changes_ui_list.append(changes_ui)
                except Exception as e:
                    logging.error(f"An error occurred in check_identify_changes: {e}")

            else:
                # Actual JSON
                actual = {}
                actual["actual"] = result_template[key]['content']
                actual["page_number"] = result_template[key]['page_num']
                actual["section_number"] = key
                actual["actual_coords"] = result_template[key]['coords']
                actual_list.append(actual)

            prev_section = current_section

    for key in result_template:
        if key not in result_agreement:
            actual = {}
            # Actual JSON
            actual["actual"] = result_template[key]['content']
            actual["page_number"] = result_template[key]['page_num']
            actual["section_number"] = key
            actual['actual_coords'] = ''
            actual_list.append(actual)

            #Changes JSON
            changes ={}
            changes["changed"] = result_template[key]['content']+' Analysis:missing'+'~!~missing'
            changes["changed_page_number"] = result_template[key]['page_num']
            changes["changed_section_number"] = key
            changes["changed_coords"] = result_template[key]['coords']
            changes_list.append(changes)

            #changes_ui JSON
            changes_ui = {}
            changes_ui["changed"] = result_template[key]['content']+' Analysis:missing'+'~!~missing'
            changes_ui["changed_page_number"] = result_template[key]['page_num']
            changes_ui["changed_section_number"] = key
            changes_ui["changed_coords"] = result_template[key]['coords']
            changes_ui_list.append(changes_ui)

    for key in result_agreement:
      if key not in result_template and key not in compared_sections:
        actual ={}
        #Actual JSON
        actual["actual"] = ""
        actual["page_number"] = result_agreement[key]['page_num']
        actual["section_number"] = key
        actual["actual_coords"] = ""
        actual_list.append(actual)
        changes = {}
        #Changes JSON
        changes["changed"] = result_agreement[key]['content']+' Analysis:New clause added'+'~!~addition'
        changes["changed_page_number"] = result_agreement[key]['page_num']
        changes["changed_section_number"] = key
        changes["changed_coords"] = result_agreement[key]['coords']
        changes_list.append(changes)

        #changes_ui JSON
        changes_ui ={}
        changes_ui["changed"] = result_agreement[key]['content']+' Analysis:New clause added'+'~!~addition'
        changes_ui["changed_page_number"] = result_agreement[key]['page_num']
        changes_ui["changed_section_number"] = key
        changes_ui["changed_coords"] = result_agreement[key]['coords']
        changes_ui_list.append(changes_ui)

    return changes_list,actual_list,changes_ui_list


deletion_prompt = """
{} : This clause has been deleted from the existing contract. what is the impact? provide me short analysis in 1 single paragraph should not exceed 100 words.
"""

addition_prompt = """
{} : This clause has been added in the existing contract. what is the impact? provide me short analysis in 1 single paragraph should not exceed 100 words.
"""


def remove_analysis(text: str) -> str:
    """
    Remove the analysis portion from the given text.

    Args:
        text (str): The text to remove the analysis from.

    Returns:
        str: The text with the analysis portion removed.
    """
    new_text = re.sub(r"(Analysis.*)", "", text)
    return new_text

def json_output(actual: List[Dict[str, str]], changes_ui: List[Dict[str, str]], file: str, template_files: str, result_template: str, result_agreement: str) -> Dict[str, any]:
    """
    Generate a JSON output based on the provided inputs.

    Args:
        actual (List[Dict[str, str]]): The list of actual changes.
        changes_ui (List[Dict[str, str]]): The list of UI changes.
        file (str): The file path.
        template_files (str): The template path.
        result_template (str): The template result.
        result_agreement (str): The agreement result.

    Returns:
        Dict[str, any]: The generated JSON output.
    """
    json_output = {}

    json_output["input_file_path"] = file
    comparison_list = []

    for i in range(len(actual)):
        actual_changes = '{}'
        actual_changes_json = json.loads(actual_changes)
        analysis_final = re.sub(r'\n', ' ', changes_ui[i]['changed'].split("~!~")[0].split("Analysis:")[1])
        if changes_ui[i]['changed'].split('~!~')[-1] == 'addition' or changes_ui[i]['changed'].split('~!~')[-1] == 'missing':
            continue
        actual_changes_json.update({"actual": actual[i]['actual']})
        actual_changes_json.update({"page_number": actual[i]['page_number']})
        actual_changes_json.update({"section_number": actual[i]['section_number']})
        actual_changes_json.update({"actual_coords": actual[i]['actual_coords']})
        actual_changes_json.update({"changed": remove_analysis(changes_ui[i]["changed"])})
        actual_changes_json.update({"changed_page_number": changes_ui[i]["changed_page_number"]})
        actual_changes_json.update({"changed_section_number": changes_ui[i]["changed_section_number"]})
        actual_changes_json.update({"changed_coords": changes_ui[i]["changed_coords"]})
        actual_changes_json.update({"analysis": analysis_final})
        actual_changes_json.update({"type_of_change": changes_ui[i]['changed'].split("~!~")[1]})
        comparison_list.append(actual_changes_json)

    for i in range(len(changes_ui)):
        actual_changes = '{}'
        actual_changes_json = json.loads(actual_changes)
        if changes_ui[i]['changed'].split('~!~')[-1] == 'missing':
            actual_changes_json.update({"actual": actual[i]['actual']})
            actual_changes_json.update({"page_number": actual[i]['page_number']})
            actual_changes_json.update({"section_number": actual[i]['section_number']})
            actual_changes_json.update({"actual_coords": actual[i]['actual_coords']})
            actual_changes_json.update({"changed": remove_analysis(changes_ui[i]["changed"])})
            actual_changes_json.update({"changed_page_number": changes_ui[i]["changed_page_number"]})
            actual_changes_json.update({"changed_section_number": changes_ui[i]["changed_section_number"]})
            actual_changes_json.update({"changed_coords": changes_ui[i]["changed_coords"]})
            final_deletion_prompt = deletion_prompt.format(actual[i]['actual'])
            actual_changes_json.update({"analysis": open_ai(final_deletion_prompt)})
            actual_changes_json.update({"type_of_change": "missing"})
            comparison_list.append(actual_changes_json)
        if changes_ui[i]['changed'].split('~!~')[-1] == 'addition':
            actual_changes_json.update({"actual": actual[i]['actual']})
            actual_changes_json.update({"page_number": actual[i]['page_number']})
            actual_changes_json.update({"section_number": actual[i]['section_number']})
            actual_changes_json.update({"actual_coords": actual[i]['actual_coords']})
            actual_changes_json.update({"changed": remove_analysis(changes_ui[i]["changed"])})
            actual_changes_json.update({"changed_page_number": changes_ui[i]["changed_page_number"]})
            actual_changes_json.update({"changed_section_number": changes_ui[i]["changed_section_number"]})
            actual_changes_json.update({"changed_coords": changes_ui[i]["changed_coords"]})
            final_addition_prompt = addition_prompt.format(changes_ui[i]["changed"])
            actual_changes_json.update({"analysis": open_ai(final_addition_prompt)})
            actual_changes_json.update({"type_of_change": "addition"})
            comparison_list.append(actual_changes_json)

    json_output["changes"] = comparison_list

    # Sort the data based on the "actual" key in ascending order
    sorted_data = sorted(json_output["changes"], key=lambda x: split_page_section(x["page_number"], x["section_number"]))

    # Update the JSON data with the sorted_data
    json_output["changes"] = sorted_data
    return json_output

def process_files_template(file: str) -> List[str]:
    """
    Process the template file and extract sections.

    Args:
        file (str): The path of the template file.

    Returns:
        List[str]: The extracted sections from the template file.
    """
    try:
        # Set the header and footer heights
        header_height = 50
        footer_height = 80

        # Extract sections from the file
        sections = extract_sections(file, header_height, footer_height)
        print("Template:",sections)
        return sections

    except Exception as e:
        # Create a logger
        logger = logging.getLogger(__name__)
        logger.error(f"Error processing template file: {str(e)}")
        raise

def process_files_original(file: str) -> List[str]:
    """
    Process the original file and extract sections.

    Args:
        file (str): The path of the original file.

    Returns:
        List[str]: The extracted sections from the original file.
    """
    try:
        # Set the header and footer heights
        header_height = 50
        footer_height = 80

        # Extract sections from the file
        sections = extract_sections(file, header_height, footer_height)
        print("Original:",sections)
        return sections

    except Exception as e:
        # Create a logger
        logger = logging.getLogger(__name__)
        logger.error(f"Error processing original file: {str(e)}")
        raise




def main_processing_function(file_path: str, template_path: str):
    """
    Processes the provided PDF file and its template to identify and report changes.

    Args:
        file_path (str): Path to the PDF file to be processed.
        template_path (str): Path to the corresponding template PDF file.

    Returns:
        dict: A dictionary containing the processed results and change analysis.
    """
    logger = logging.getLogger(__name__)

    try:
        # Process the original PDF file
        result_agreement = process_files_original(file_path)

        # Process the template PDF file
        result_template = process_files_template(template_path)

        # Compare the sections extracted from the original and template files
        changes, actual, changes_ui = process_comparisons(result_agreement, result_template)

        # Generate a JSON output summarizing the changes
        final_output = json_output(actual, changes_ui, file_path, template_path, result_template, result_agreement)

        return final_output

    except Exception as e:
        logger.error(f"Error processing files: {e}")
        raise Exception(f"Error during file processing: {str(e)}")


# Streamlit UI integration for the application


def main():
    st.set_page_config(layout="wide")  # Set the layout to wide mode
    st.title('PDF Document Processor')

    # File uploaders for the agreement and template documents
    uploaded_agreement = st.file_uploader("Upload the PDF Agreement", type=['pdf'])
    uploaded_template = st.file_uploader("Upload the PDF Template", type=['pdf'])

    if uploaded_agreement and uploaded_template:
        # Save the uploaded files temporarily for processing
        with NamedTemporaryFile(delete=False, suffix=".pdf", mode='wb') as temp_agreement:
            temp_agreement.write(uploaded_agreement.read())
            agreement_path = temp_agreement.name

        with NamedTemporaryFile(delete=False, suffix=".pdf", mode='wb') as temp_template:
            temp_template.write(uploaded_template.read())
            template_path = temp_template.name

        # Process the files and display the results
        try:
            result = main_processing_function(agreement_path, template_path)
            st.success("Files successfully processed!")

            # Convert the result dictionary to a DataFrame
            df_changes = pd.DataFrame(result['changes'])
            df_changes = df_changes[['section_number', 'page_number', 'actual', 'changed', 'analysis', 'type_of_change']]

            # Display the DataFrame in the UI
            st.dataframe(df_changes, height=600)  # You can adjust height based on your needs

            # Convert DataFrame to CSV for download
            csv = df_changes.to_csv(index=False)
            b64 = base64.b64encode(csv.encode()).decode()  # some browsers need base64 encoding
            href = f'<a href="data:file/csv;base64,{b64}" download="document_changes.csv">Download CSV File</a>'
            st.markdown(href, unsafe_allow_html=True)

        except Exception as e:
            st.error(f"Error processing files: {e}")
        finally:
            # Clean up temporary files after processing
            os.remove(agreement_path)
            os.remove(template_path)

if __name__ == "__main__":
    main()