findUpdate.py

import re
import os
import json
import time
import fitz
import difflib
import secrets
import openai
import string
import logging
import requests
import streamlit as st
from pandas import DataFrame
from openai import AzureOpenAI
from difflib import SequenceMatcher
from tempfile import NamedTemporaryFile
from typing import Tuple, Dict, List, Union
from diff_match_patch import diff_match_patch
import base64
import pandas as pd

class FindUpdate:
    @staticmethod
    def clean_string(text: str, stem: str = "None") -> str:
        """Clean the input text by removing punctuation, numbers, and optionally stemming or lemmatizing the words.

        Args:
            text (str): The input text to be cleaned.
            stem (str, optional): The stemming method to be used. Options are "None" (default), "Stem", "Lem", or "Spacy".

        Returns:
            str: The cleaned text.

        Raises:
            None
        """
        try:
            final_string = ""
            # Replace any characters with nothing or a space
            text = re.sub(r'\n', ' ', text)
            text = re.sub(r' +', ' ', text)
            text = re.sub(r'[^\x00-\x7f]',r'', text)
            # Remove punctuation
            translator = str.maketrans('', '', string.punctuation)
            text = text.translate(translator)
            # Remove numbers
            text_filtered = [re.sub(r'\w*\d\w*\s+', '', w) for w in text]
            text_filtered = [re.sub('[0-9]', '', w) for w in text_filtered]
            # Stem or Lemmatize
            if stem == 'Stem':
                stemmer = PorterStemmer()
                text_stemmed = [stemmer.stem(y) for y in text_filtered]
            elif stem == 'Lem':
                lem = WordNetLemmatizer()
                text_stemmed = [lem.lemmatize(y) for y in text_filtered]
            elif stem == 'Spacy':
                text_filtered = nlp(' '.join(text_filtered))
                text_stemmed = [y.lemma_ for y in text_filtered]
            else:
                text_stemmed = text_filtered

            final_string = ''.join(text_stemmed)
        except Exception as e:
            # Handle the exception here
            logging.error(f"An error occurred in clean_string: {e}")
            return None

        return final_string
    @staticmethod
    def extract_text_without_header_footer(page: fitz.Page, page_height: float, header_height: float, footer_height: float) -> str:
        """
        Extract the text from a page of a PDF document, excluding the header and footer.

        Args:
            page (fitz.Page): The page object from the PyMuPDF library.
            page_height (float): The height of the page.
            header_height (float): The height of the header to be excluded.
            footer_height (float): The height of the footer to be excluded.

        Returns:
            str: The extracted text.

        Raises:
            None
        """
        try:
            exclude_top = header_height
            exclude_bottom = page_height - footer_height
            text = ""
            for text_block in page.get_text_blocks():
                bbox = fitz.Rect(text_block[:4])  # Get bounding box of text block
                if bbox.y0 >= exclude_top and bbox.y1 <= exclude_bottom:
                    text += text_block[4] + "\n"  # Add text content to result
            return text
        except Exception as e:
            # Handle the exception here
            logging.error(f"An error occurred in extract_text_without_header_footer: {e}")
            return ""
    @staticmethod
    def rect_to_dict(rect: fitz.Rect) -> Dict[str, float]:
        """
        Convert a PyMuPDF Rect object to a dictionary.

        Args:
            rect (fitz.Rect): The Rect object from the PyMuPDF library.

        Returns:
            Dict[str, float]: The dictionary representation of the Rect object.

        Raises:
            None
        """
        return {
            "x1": rect[0],
            "y1": rect[1],
            "x2": rect[2],
            "y2": rect[3]
        }
    @staticmethod
    def extract_sections(pdf_path: str, header_height: float, footer_height: float) -> Dict[str, Dict[str, List[str]]]:
        """
        Extract sections from a PDF document, excluding the header and footer.

        Args:
            pdf_path (str): The path to the PDF document.
            header_height (float): The height of the header to be excluded.
            footer_height (float): The height of the footer to be excluded.

        Returns:
            Dict[str, Dict[str, List[str]]]: A dictionary containing the extracted sections.

        Raises:
            None
        """
        try:
            doc = fitz.open(pdf_path)
            sections = {}
            current_section = None
            for page_num in range(2, len(doc)):
                page = doc.load_page(page_num)
                page_height = page.rect.height
                text = FindUpdate.extract_text_without_header_footer(page, page_height, header_height, footer_height)
                lines = text.split('\n')
                print("start")
                for line in lines:
                    match = re.match(r'^(?:\d+\.\d*(?:\.\d+)*)|AT&T ALLIANCE PROGRAM AGREEMENT', line.strip())
                    if match:
                        print("line:",line)
                        section_num = match.group()
                        if section_num != current_section:
                            current_section = section_num
                            sections[current_section] = {'page_num': str(page_num + 1), 'coords': [], 'content': []}

                    if current_section:
                        sections[current_section]['content'].append(line)
                        bbox = page.search_for(line)
                        if bbox:
                            for cord in bbox:
                                sections[current_section]['coords'].append(FindUpdate.rect_to_dict(cord))
            print("section-keys:",sections.keys())
            for key in sections.keys():
                if 'content' in sections[key]:
                    sections[key]['content'] = (' '.join(sections[key]['content']))[len(key):].lstrip()
            print("Content:",sections)
            return sections
        except Exception as e:
            # Handle the exception here
            logging.error(f"An error occurred in extract_sections: {e}")
            return {}
    @staticmethod
    def check_identify_changes(template_text: str, contract_text: str) -> str:
        """
        Compare the template text with the contract text and identify the changes made.

        Args:
            template_text (str): The template text.
            contract_text (str): The contract text.

        Returns:
            str: A sentence prompt describing the changes made.

        Raises:
            None
        """
        try:
            dmp = diff_match_patch()
            diff = dmp.diff_main(template_text, contract_text)
            dmp.diff_cleanupSemantic(diff)

            if  all([True if each[0] == 0 or each[1] == ' ' else False for each in diff]):
                return('no_change')
            else:
                deleted = []
                added = []
                for each in diff:
                    if each[0] == -1:
                        deleted.append(each[1])
                    elif each[0] == 1:
                        added.append(each[1])
                sentence_prompt = f"""
                template text = {template_text}
                contract_text = {contract_text}
                deleted from template --- {deleted}
                added to the actual contract -- {added}
                """
                return(sentence_prompt)
        except Exception as e:
        # Handle the exception here
            logging.error(f"An error occurred in check_identify_changes: {e}")
            return ""
    @staticmethod
    def open_ai(prompt: str) -> str:
        """
        Generate AI response using OpenAI GPT-4 model.

        Args:
            prompt (str): The prompt for the AI model.

        Returns:
            str: The generated AI response.

        Raises:
            None
        """
        stat = 0
        while stat == 0:
            try:
                client = AzureOpenAI(api_key="5c68e66b7eb3470987c9088b6602c6c7",
                        api_version="2023-07-01-preview",
                        azure_endpoint="https://azureadople.openai.azure.com/")
                conversation = []
                conversation.append({"role": "user", "content": prompt})
                response = client.chat.completions.create(
                    model="GPT-3",
                    messages=conversation,
                    temperature=0,
                    max_tokens=3000,
                    stop=None
                )
                stat = 1
            except openai.RateLimitError as e:
                logging.error(f"Rate limit error occurred: {e}")
                stat = 0
                time.sleep(60)
            except Exception as e:
                logging.error(f"An error occurred in open_ai: {e}")
                stat = 0

        output = response.choices[0].message.content
        return output

    @staticmethod
    def compare_strings(string1: str, string2: str) -> float:
        """
        Compare two strings and return their similarity ratio.

        Args:
            string1 (str): The first string.
            string2 (str): The second string.

        Returns:
            float: The similarity ratio between the two strings.

        Raises:
            None
        """
        matcher = SequenceMatcher(None, string1, string2)
        similarity_ratio = matcher.ratio()
        return similarity_ratio
    @staticmethod
    def get_main_section(section_number: str) -> str:
        """
        Get the main section number from a given section number.

        Args:
            section_number (str): The section number.

        Returns:
            str: The main section number.

        Raises:
            None+
        """
        parts = section_number.split('.')
        if len(parts) > 1:
            parts.pop()

        return ".".join(parts)
    @staticmethod
    def increment_section(section_number: str) -> str:
        """
        Increment a section number by one.

        Args:
            section_number (str): The section number.

        Returns:
            str: The incremented section number.

        Raises:
            None
        """
        parts = section_number.split('.')
        last_part = '0' if not parts[-1] else parts[-1]  # Get the last part of the section number
        # Check if the last part is numeric
        if last_part.isdigit():
            # Convert the last part to an integer and increment it by one
            incremented_last_part = str(int(last_part) + 1)
            # Replace the last part in the parts list
            parts[-1] = incremented_last_part
        else:
            # If the last part is not numeric, return the original section number
            return section_number

        # Join the parts back together with '.'
        incremented_section = '.'.join(parts)
        return incremented_section
    @staticmethod
    def count_subsections(dictionary: Dict[str, Union[str, Dict]]) -> Dict[str, int]:
        """
        Count the number of subsections in a dictionary.

        Args:
            dictionary (Dict[str, Union[str, Dict]]): The dictionary containing the subsections.

        Returns:
            Dict[str, int]: A dictionary with the main section numbers as keys and the count of subsections as values.

        Raises:
            None
        """
        section_counts = {}
        for key in dictionary:
            # Split the key by '.' and take the first part as the section number
            section_number = FindUpdate.get_main_section(key)

            # Check if the section number is already in the section_counts dictionary
            if section_number in section_counts:
                # Increment the count for the section
                section_counts[section_number] += 1
            else:
                # Initialize the count for the section
                section_counts[section_number] = 1

        return section_counts
    @staticmethod
    def split_page_section(page_no: str, sec_no: str) -> Tuple[str, Tuple[int]]:
        """
        Split the page number and section number into separate components.

        Args:
            page_no (str): The page number.
            sec_no (str): The section number.

        Returns:
            Tuple[str, Tuple[int]]: A tuple containing the page number and section number.

        Raises:
            None
        """
        try:
            page_number = page_no
            section_numbers = sec_no
            section_numbers = [int(num) if num and num.isdigit() else 0 for num in section_numbers]
            section_number = tuple(section_numbers)
            return (page_number, section_number)
        except Exception as e:
            logging.error(f"An error occurred in split_page_section: {e}")
            return ("", ())
    @staticmethod
    def process_comparisons(result_agreement: Dict[str, Dict], result_template: Dict[str, Dict]) -> Tuple[List[Dict], List[Dict], List[Dict]]:
        """
        Process the comparisons between the agreement and template texts.

        Args:
            result_agreement (Dict[str, Dict]): The agreement text.
            result_template (Dict[str, Dict]): The template text.

        Returns:
            Tuple[List[Dict], List[Dict], List[Dict]]: A tuple containing the changes list, actual list, and changes_ui list.

        Raises:
            None
        """
        # Initialize a flag to track if there are any changes
        changes = {}
        changes_ui = {}
        actual = {}
        compared_sections = []
        actual_list = []
        changes_list = []
        changes_ui_list = []
        sections_added = False
        current_section = 0
        prev_section = 0

        # Iterate through the keys and values of the dictionaries
        for key in result_template:
            current_section = FindUpdate.get_main_section(key)
            if sections_added and current_section == prev_section:
                contract_key = FindUpdate.increment_section(current_section)
            else:
                contract_key = key

            if prev_section != current_section:
                sections_added = False

            if contract_key in result_agreement:
                if FindUpdate.count_subsections(result_agreement)[current_section] != FindUpdate.count_subsections(result_template)[current_section]:
                    if FindUpdate.compare_strings(result_template[key]['content'], result_agreement[contract_key]['content']) < .1:
                        # Actual JSON
                        actual = {}
                        actual["actual"] = ""
                        actual["page_number"] = result_template[key]['page_num']
                        actual["section_number"] = key
                        actual["actual_coords"] = ""
                        actual_list.append(actual)
                        # Changes JSON
                        changes = {}
                        changes["changed"] = result_agreement[contract_key]['content'] + ' Analysis:New clause added' + '~!~addition'
                        changes["changed_page_number"] = result_agreement[contract_key]['page_num']
                        changes["changed_section_number"] = key
                        changes["changed_coords"] = result_agreement[contract_key]['coords']
                        changes_list.append(changes)
                        # changes_ui JSON
                        changes_ui = {}
                        changes_ui["changed"] = result_agreement[contract_key]['content'] + ' Analysis:New clause added' + '~!~addition'
                        changes_ui["changed_page_number"] = result_agreement[contract_key]['page_num']
                        changes_ui["changed_section_number"] = key
                        changes_ui["changed_coords"] = result_agreement[contract_key]['coords']
                        changes_ui_list.append(changes_ui)

                        contract_key = FindUpdate.increment_section(contract_key)
                        sections_added = True

                compared_sections.append(contract_key)

                if contract_key in result_agreement:
                    try:
                        prompt = f""""
As an attorney representing your client, your task is to compare the template text and the contract text provided and identify any changes that may impact the agreement or the involved parties. Specifically, you need to analyze the legal implications and considerations of the word 'subcontractors' when it appears in the changed text. Instead of focusing solely on the addition or removal of the letters 'or,' provide a comprehensive analysis based on the complete word 'subcontractors.' Classify the changes as either "minor_change" or "major_change."
Please provide the changed text alone as a separate paragraph under the "Changed:" subheading, and the analysis of the changes as a separate paragraph under the "Analysis:" subheading. At the end, add ~!~ and classification like ~!~minor_change or ~!~major_change.
EXAMPLES -
1.
Contract text:
Verify identification credentials including Social Security number, driver’s license, educational credentials, employment history, home address, and citizenship indicia;
In connection with providing access to a customer’s facilities or systems, comply with any additional investigation or screening requirements required by such customer as communicated in advance; test for use of illicit drugs including opiates, cocaine, cannabinoids, amphetamines, and phencyclidine.
Template text:
Verify identification credentials including Social Security number, driver’s license, educational credentials, employment history, home address, and citizenship indicia.
In connection with providing access to a customer’s facilities or systems, comply with any additional investigation or screening requirements required by such customer as communicated in advance. Test for use of illicit drugs including opiates, cocaine, cannabinoids, amphetamines, and phencyclidine.
Differences Observed -
Changed: "When (i) the customer or end user is a federal, state, or local government entity"
Analysis:
A new sentence is added: "When (i) the customer or end user is a federal, state, or local government entity" in the template text. This clause specifies a condition under which the drug testing requirement applies. In the template text, this condition is excluded, meaning that the drug testing requirement would apply regardless of whether the customer or end user is a government entity. This could potentially change the scope and applicability of the drug testing provision.~!~major_change
2.
Contract text:
Perform a criminal background check to determine, in the counties, states, and federal court districts where the candidate has lived, worked, or attended school in the previous ten years, whether the candidate has been: (1) convicted of any felony; (2) convicted of a misdemeanor involving violence, theft, computer crimes, fraud, financial crimes, drug distribution, unlawful possession or use of a dangerous weapon, or sexual misconduct; or, (3) listed on any government registry as a sex offender (together, “Conviction”); and in connection with providing access to a customer’s facilities or systems, comply with any additional investigation or screening requirements required by such customer as communicated in advance.
Template text:
Perform a criminal background check to determine, in the counties, states, and federal court districts where the candidate has lived, worked, or attended school in the previous ten years, whether the candidate has been: (1) convicted of any felony; (2) convicted of a misdemeanor involving violence, theft, computer crimes, fraud, financial crimes, drug distribution, unlawful possession or use of a dangerous weapon, or sexual misconduct; or, (3) listed on any government registry as a sex offender (together, “Conviction”). In connection with providing access to a customer’s facilities or systems, comply with any additional investigation or screening requirements required by such customer as communicated in advance.
Differences Observed -
Changed: "and"
Analysis:
The contract text includes the word "and," which isn't in the template. This could potentially change the interpretation of the agreement, as it could be read as each clause being a separate requirement, rather than a list of requirements.~!~major_change
3.
Contract text:
The service provider may not assign, delegate, or otherwise transfer its rights or obligations under this Agreement, voluntarily or involuntarily, without the prior written consent of the other party except that the service provider may delegate certain obligations hereunder to its subcontractors as contemplated by this Agreement. Any attempted assignment, delegation, or transfer not consented to in writing will be void. Notwithstanding the foregoing, with notice to the other party, either party may assign this Agreement, in whole or in part, to any affiliate, successor-in-interest or without securing the consent of the other party. Any assignment of money will be void if (i) the assignor fails to give the non-assigning party at least thirty (30) days prior written notice, or (ii) the assignment purports to impose upon the non-assigning party additional costs or obligations in addition to the payment of such money, or (iii) the assignment purports to preclude the other party from dealing solely and directly with the service provider in all matters pertaining to this Agreement. This Agreement binds and benefits both parties and their permitted successors and assigns.
Template text:
The service provider may not assign, delegate, or otherwise transfer its rights or obligations under this Agreement, voluntarily or involuntarily, without the prior written consent of the other party except that the service provider may delegate certain obligations hereunder to its subcontracts as contemplated by this Agreement. Any attempted assignment, delegation, or transfer not consented to in writing will be void. Notwithstanding the foregoing, with notice to the other party, either party may assign this Agreement, in whole or in part, to any affiliate, successor-in-interest or without securing the consent of the other party. Any assignment of money will be void if (i) the assignor fails to give the non-assigning party at least thirty (30) days prior written notice, or (ii) the assignment purports to impose upon the non-assigning party additional costs or obligations in addition to the payment of such money, or (iii) the assignment purports to preclude the other party from dealing solely and directly with the service provider in all matters pertaining to this Agreement. This Agreement binds and benefits both parties and their permitted successors and assigns.
Differences Observed -
Changed: "Subcontractors"
Analysis: The contract text includes the word "Subcontractors" instead of "Subcontracts" as in the template. This change could potentially alter the meaning of the agreement as it specifies a different entity (Subcontractors vs Subcontracts). However, given the context, it seems likely that this is a typographical error in the template text, and the intended meaning remains the same.~!~minor_change
4.
Contract text:
This AGREEMENT is entered into between [Company Name], a [State] corporation, which sometimes does business as [DBA Name], (“Company”) and [Other Party Name], a [State] LLC (“Solution Provider” or “SP”). The company and solution provider may be referred to collectively as the “Parties” or individually as a “Party.”
Template text:
ALLIANCE PROGRAM AGREEMENT This AGREEMENT is entered into between [Company Name], a [State] corporation, which sometimes does business as [DBA Name], (“Company”) and ________________________, a __________ corporation (“Solution Provider” or “SP”). The company and solution provider may be referred to collectively as the “Parties” or individually as a “Party.”
Differences Observed -
Changed:"[Other Party Name], a [State] LLC"
Analysis: The contract text includes the specific name of the corporation "[Other Party Name], a [State] LLC" instead of a placeholder as in the template. This is a minor change as it is expected that the specific name of the corporation would be filled in the actual contract.~!~minor_change
"""
                        res = FindUpdate.check_identify_changes(FindUpdate.clean_string(result_template[key]['content']), FindUpdate.clean_string(result_agreement[contract_key]['content']))
                        if res != 'no_change':
                            final_prompt =prompt + res
                            llm_res = FindUpdate.open_ai(final_prompt)
                            # Actual JSON
                            actual = {}
                            actual["actual"] = result_template[key]['content']
                            actual["page_number"] = result_template[key]['page_num']
                            actual["section_number"] = key
                            actual["actual_coords"] = result_template[key]['coords']
                            actual_list.append(actual)
                            # Changes JSON
                            changes = {}
                            changes["changed"] = llm_res
                            changes["changed_page_number"] = result_agreement[contract_key]['page_num']
                            changes["changed_section_number"] = key
                            changes["changed_coords"] = result_agreement[contract_key]['coords']
                            changes_list.append(changes)
                            # changes_ui JSON
                            changes_ui = {}
                            changes_ui["changed"] = llm_res
                            changes_ui["changed_page_number"] = result_agreement[contract_key]['page_num']
                            changes_ui["changed_section_number"] = key
                            changes_ui["changed_coords"] = result_agreement[contract_key]['coords']
                            changes_ui_list.append(changes_ui)
                    except Exception as e:
                        logging.error(f"An error occurred in check_identify_changes: {e}")

                else:
                    # Actual JSON
                    actual = {}
                    actual["actual"] = result_template[key]['content']
                    actual["page_number"] = result_template[key]['page_num']
                    actual["section_number"] = key
                    actual["actual_coords"] = result_template[key]['coords']
                    actual_list.append(actual)

                prev_section = current_section

        for key in result_template:
            if key not in result_agreement:
                actual = {}
                # Actual JSON
                actual["actual"] = result_template[key]['content']
                actual["page_number"] = result_template[key]['page_num']
                actual["section_number"] = key
                actual['actual_coords'] = ''
                actual_list.append(actual)

                #Changes JSON
                changes ={}
                changes["changed"] = result_template[key]['content']+' Analysis:missing'+'~!~missing'
                changes["changed_page_number"] = result_template[key]['page_num']
                changes["changed_section_number"] = key
                changes["changed_coords"] = result_template[key]['coords']
                changes_list.append(changes)

                #changes_ui JSON
                changes_ui = {}
                changes_ui["changed"] = result_template[key]['content']+' Analysis:missing'+'~!~missing'
                changes_ui["changed_page_number"] = result_template[key]['page_num']
                changes_ui["changed_section_number"] = key
                changes_ui["changed_coords"] = result_template[key]['coords']
                changes_ui_list.append(changes_ui)

        for key in result_agreement:
            if key not in result_template and key not in compared_sections:
                actual ={}
                #Actual JSON
                actual["actual"] = ""
                actual["page_number"] = result_agreement[key]['page_num']
                actual["section_number"] = key
                actual["actual_coords"] = ""
                actual_list.append(actual)
                changes = {}
                #Changes JSON
                changes["changed"] = result_agreement[key]['content']+' Analysis:New clause added'+'~!~addition'
                changes["changed_page_number"] = result_agreement[key]['page_num']
                changes["changed_section_number"] = key
                changes["changed_coords"] = result_agreement[key]['coords']
                changes_list.append(changes)

                #changes_ui JSON
                changes_ui ={}
                changes_ui["changed"] = result_agreement[key]['content']+' Analysis:New clause added'+'~!~addition'
                changes_ui["changed_page_number"] = result_agreement[key]['page_num']
                changes_ui["changed_section_number"] = key
                changes_ui["changed_coords"] = result_agreement[key]['coords']
                changes_ui_list.append(changes_ui)

            return changes_list,actual_list,changes_ui_list

    @staticmethod
    def remove_analysis(text: str) -> str:
        """
        Remove the analysis portion from the given text.

        Args:
            text (str): The text to remove the analysis from.

        Returns:
            str: The text with the analysis portion removed.
        """
        new_text = re.sub(r"(Analysis.*)", "", text)
        return new_text
    @staticmethod
    def json_output(actual: List[Dict[str, str]], changes_ui: List[Dict[str, str]], file: str, template_files: str, result_template: str, result_agreement: str) -> Dict[str, any]:
        """
        Generate a JSON output based on the provided inputs.

        Args:
            actual (List[Dict[str, str]]): The list of actual changes.
            changes_ui (List[Dict[str, str]]): The list of UI changes.
            file (str): The file path.
            template_files (str): The template path.
            result_template (str): The template result.
            result_agreement (str): The agreement result.

        Returns:
            Dict[str, any]: The generated JSON output.
        """
        deletion_prompt = """
        {} : This clause has been deleted from the existing contract. what is the impact? provide me short analysis in 1 single paragraph should not exceed 100 words.
        """

        addition_prompt = """
        {} : This clause has been added in the existing contract. what is the impact? provide me short analysis in 1 single paragraph should not exceed 100 words.
        """
        json_output = {}

        json_output["input_file_path"] = file
        comparison_list = []

        for i in range(len(actual)):
            actual_changes = '{}'
            actual_changes_json = json.loads(actual_changes)
            analysis_final = re.sub(r'\n', ' ', changes_ui[i]['changed'].split("~!~")[0].split("Analysis:")[1])
            if changes_ui[i]['changed'].split('~!~')[-1] == 'addition' or changes_ui[i]['changed'].split('~!~')[-1] == 'missing':
                continue
            actual_changes_json.update({"actual": actual[i]['actual']})
            actual_changes_json.update({"page_number": actual[i]['page_number']})
            actual_changes_json.update({"section_number": actual[i]['section_number']})
            actual_changes_json.update({"actual_coords": actual[i]['actual_coords']})
            actual_changes_json.update({"changed": FindUpdate.remove_analysis(changes_ui[i]["changed"])})
            actual_changes_json.update({"changed_page_number": changes_ui[i]["changed_page_number"]})
            actual_changes_json.update({"changed_section_number": changes_ui[i]["changed_section_number"]})
            actual_changes_json.update({"changed_coords": changes_ui[i]["changed_coords"]})
            actual_changes_json.update({"analysis": analysis_final})
            actual_changes_json.update({"type_of_change": changes_ui[i]['changed'].split("~!~")[1]})
            comparison_list.append(actual_changes_json)

        for i in range(len(changes_ui)):
            actual_changes = '{}'
            actual_changes_json = json.loads(actual_changes)
            if changes_ui[i]['changed'].split('~!~')[-1] == 'missing':
                actual_changes_json.update({"actual": actual[i]['actual']})
                actual_changes_json.update({"page_number": actual[i]['page_number']})
                actual_changes_json.update({"section_number": actual[i]['section_number']})
                actual_changes_json.update({"actual_coords": actual[i]['actual_coords']})
                actual_changes_json.update({"changed": FindUpdate.remove_analysis(changes_ui[i]["changed"])})
                actual_changes_json.update({"changed_page_number": changes_ui[i]["changed_page_number"]})
                actual_changes_json.update({"changed_section_number": changes_ui[i]["changed_section_number"]})
                actual_changes_json.update({"changed_coords": changes_ui[i]["changed_coords"]})
                final_deletion_prompt = deletion_prompt.format(actual[i]['actual'])
                actual_changes_json.update({"analysis": FindUpdate.open_ai(final_deletion_prompt)})
                actual_changes_json.update({"type_of_change": "missing"})
                comparison_list.append(actual_changes_json)
            if changes_ui[i]['changed'].split('~!~')[-1] == 'addition':
                actual_changes_json.update({"actual": actual[i]['actual']})
                actual_changes_json.update({"page_number": actual[i]['page_number']})
                actual_changes_json.update({"section_number": actual[i]['section_number']})
                actual_changes_json.update({"actual_coords": actual[i]['actual_coords']})
                actual_changes_json.update({"changed": FindUpdate.remove_analysis(changes_ui[i]["changed"])})
                actual_changes_json.update({"changed_page_number": changes_ui[i]["changed_page_number"]})
                actual_changes_json.update({"changed_section_number": changes_ui[i]["changed_section_number"]})
                actual_changes_json.update({"changed_coords": changes_ui[i]["changed_coords"]})
                final_addition_prompt =addition_prompt.format(changes_ui[i]["changed"])
                actual_changes_json.update({"analysis": FindUpdate.open_ai(final_addition_prompt)})
                actual_changes_json.update({"type_of_change": "addition"})
                comparison_list.append(actual_changes_json)

        json_output["changes"] = comparison_list

        # Sort the data based on the "actual" key in ascending order
        sorted_data = sorted(json_output["changes"], key=lambda x: FindUpdate.split_page_section(x["page_number"], x["section_number"]))

        # Update the JSON data with the sorted_data
        json_output["changes"] = sorted_data
        return json_output
    @staticmethod
    def process_files_template(file: str) -> List[str]:
        """
        Process the template file and extract sections.

        Args:
            file (str): The path of the template file.

        Returns:
            List[str]: The extracted sections from the template file.
        """
        try:
            # Set the header and footer heights
            header_height = 50
            footer_height = 80

            # Extract sections from the file
            sections = FindUpdate.extract_sections(file, header_height, footer_height)
            print("Template:",sections)
            return sections

        except Exception as e:
            # Create a logger
            logger = logging.getLogger(__name__)
            logger.error(f"Error processing template file: {str(e)}")
            raise
    @staticmethod
    def process_files_original(file: str) -> List[str]:
        """
        Process the original file and extract sections.

        Args:
            file (str): The path of the original file.

        Returns:
            List[str]: The extracted sections from the original file.
        """
        try:
            # Set the header and footer heights
            header_height = 50
            footer_height = 80

            # Extract sections from the file
            sections = FindUpdate.extract_sections(file, header_height, footer_height)
            print("Original:",sections)
            return sections

        except Exception as e:
            # Create a logger
            logger = logging.getLogger(__name__)
            logger.error(f"Error processing original file: {str(e)}")
            raise

    @staticmethod
    def main_processing_function(file_path: str, template_path: str):
        """
        Processes the provided PDF file and its template to identify and report changes.

        Args:
            file_path (str): Path to the PDF file to be processed.
            template_path (str): Path to the corresponding template PDF file.

        Returns:
            dict: A dictionary containing the processed results and change analysis.
        """
        logger = logging.getLogger(__name__)

        try:
            # Process the original PDF file
            result_agreement = FindUpdate.process_files_original(file_path)

            # Process the template PDF file
            result_template = FindUpdate.process_files_template(template_path)

            # Compare the sections extracted from the original and template files
            changes, actual, changes_ui = FindUpdate.process_comparisons(result_agreement, result_template)

            # Generate a JSON output summarizing the changes
            final_output = FindUpdate.json_output(actual, changes_ui, file_path, template_path, result_template, result_agreement)

            return final_output

        except Exception as e:
            logger.error(f"Error processing files: {e}")
            raise Exception(f"Error during file processing: {str(e)}")