use without pronoun model

anonymize_file_clear.py

@@ -7,9 +7,9 @@ import gensim
 from concrete.ml.common.serialization.loaders import load
 
 def load_models():
-    base_dir = Path(__file__).parent
-    embeddings_model = gensim.models.FastText.load(str(base_dir / "embedded_model.model"))
-    with open(base_dir / "cml_xgboost.model", "r") as model_file:
+    base_dir = Path(__file__).parent / "models"
+    embeddings_model = gensim.models.FastText.load(str(base_dir / "without_pronoun_embedded_model.model"))
+    with open(base_dir / "without_pronoun_cml_xgboost.model", "r") as model_file:
         fhe_ner_detection = load(file=model_file)
     return embeddings_model, fhe_ner_detection
 
@@ -34,7 +34,8 @@ def anonymize_text(text, embeddings_model, fhe_ner_detection):
         else:
             processed_tokens.append(token)  # Preserve punctuation and spaces as is
 
-    return uuid_map
+    anonymized_text = ''.join(processed_tokens)
+    return anonymized_text, uuid_map
 
 def main():
     parser = argparse.ArgumentParser(description="Anonymize named entities in a text file and save the mapping to a JSON file.")
@@ -47,14 +48,26 @@ def main():
     with open(args.file_path, 'r', encoding='utf-8') as file:
         text = file.read()
 
+    # Save the original text to its specified file
+    original_file_path = Path(__file__).parent / "files" / "original_document.txt"
+    with open(original_file_path, 'w', encoding='utf-8') as original_file:
+        original_file.write(text)
+    
     # Anonymize the text
-    uuid_map = anonymize_text(text, embeddings_model, fhe_ner_detection)
+    anonymized_text, uuid_map = anonymize_text(text, embeddings_model, fhe_ner_detection)
+
+    # Save the anonymized text to its specified file
+    anonymized_file_path = Path(__file__).parent / "files" / "anonymized_document.txt"
+    with open(anonymized_file_path, 'w', encoding='utf-8') as anonymized_file:
+        anonymized_file.write(anonymized_text)
 
     # Save the UUID mapping to a JSON file
     mapping_path = Path(args.file_path).stem + "_uuid_mapping.json"
     with open(mapping_path, 'w', encoding='utf-8') as file:
         json.dump(uuid_map, file, indent=4, sort_keys=True)
 
+    print(f"Original text saved to {original_file_path}")
+    print(f"Anonymized text saved to {anonymized_file_path}")
     print(f"UUID mapping saved to {mapping_path}")
 
 if __name__ == "__main__":

app.py

@@ -59,13 +59,13 @@ def query_chatgpt(anonymized_query):
     tokens = re.findall(token_pattern, anonymized_response)
     processed_tokens = []
 
-    print(tokens)
+
     for token in tokens:
         # Directly append non-word tokens or whitespace to processed_tokens
         if not token.strip() or not re.match(r"\w+", token):
             processed_tokens.append(token)
             continue
-        print(token)
+
         if token in inverse_uuid_map:
             processed_tokens.append(inverse_uuid_map[token])
         else:
@@ -136,7 +136,7 @@ with demo:
 
         anonymized_text_output = gr.Textbox(label="Anonymized Text with FHE", lines=13)
 
-    identified_words_output = gr.Dataframe(label="Identified Words", visible=False)
+    identified_words_output = gr.Dataframe(label="Identified Words", visible=True)
 
     submit_button = gr.Button("Anonymize with FHE")
 

demo_text.txt

@@ -1 +1 @@
-Who lives in Maine?
+who lives in Maine?

deployment/client.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ef07d7867a023a0d3aeab3d029b5ab4941786841fee38ce993b38af00392142
-size 253880
+oid sha256:2e4798bf93e38a14f5f1aa15203bb093cf15c4dfee7edbd8e0f7767605755090
+size 129876

deployment/server.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c446388b3e562c5babd8f98d00229e78d5c46952764b2bf577ca70bd5d96cb0
-size 24234
+oid sha256:86fc185025bf0c84d50aef44f14308c0077db2701a9a835bf4e3b912a58cb9b0
+size 5639

fhe_anonymizer.py

@@ -13,10 +13,10 @@ class FHEAnonymizer:
     def __init__(self, punctuation_list=".,!?:;"):
 
         self.embeddings_model = gensim.models.FastText.load(
-            str(base_dir / "embedded_model.model")
+            str(base_dir / "models/without_pronoun_embedded_model.model")
         )
         self.punctuation_list = punctuation_list
-        with open(base_dir / "cml_xgboost.model", "r") as model_file:
+        with open(base_dir / "models/without_pronoun_cml_xgboost.model", "r") as model_file:
             self.fhe_ner_detection = load(file=model_file)
 
         with open(base_dir / "original_document_uuid_mapping.json", 'r') as file:
@@ -50,7 +50,8 @@ class FHEAnonymizer:
 
             # Prediction for each word
             x = self.embeddings_model.wv[token][None]
-            prediction_proba = self.fhe_ner_detection.predict_proba(x)
+            # prediction_proba = self.fhe_ner_detection.predict_proba(x)
+            prediction_proba = self.fhe_inference(x)
             probability = prediction_proba[0][1]
 
             if probability >= 0.5:

files/anonymized_document.txt

@@ -1,10 +1,10 @@
-Hello, 0a182475 name is 84e24eb8 0ea0e35c and 9f2963af live in 5d0a593d.
-6d7a71e0 credit card number is 1155d2a5 and 0a182475 d8e7c7fc c627b841 id is be0eddce.
+Hello, my name is 97dc4202 7ce27ecb and I live in aaf4b006.
+My credit card number is bfd59a59 and my c7184a17 516361a1 f8380bf5 is edf660df.
 
-On b926f540 ee1fa38f 9f2963af visited 1e63e774 and sent an email to f70cf334,  from the IP 2d66aefe.
+On d615d819 3f343449 I visited b6394fb9 and 732237ac an email to a295c5d0,  from the c591bc5d c83dd929.
 
-6d7a71e0 passport: 53b52085 and 0a182475 phone number: 81133b16 08aeb9f5.
+My passport: c263d176 and my c402f998 number: a054c8c2 8fddc160.
 
-3bbcaf02 is a valid 11e27c41 5a60bcfa 3730179e 97008100: ab18e6bb . Can 773b08cb f4953628 check the 8ff83dc3 on bank account e1a23ada?
+This is a 08876c6f c5462fed 49b9cffb 3658044b Number: 2f075e1d . Can you please e51c8e1c the f1b9c36f on 4fd4e4c4 aa960526 148fea84?
 
-7ab7335c's social security number is 11e90148.  67e5e61e driver license? it is c6652117.
+a18f3dda's bcda6774 security number is 48f7c8a4.  Her driver license? it is ab7ec0c3.

files/question_demo.txt

@@ -1,10 +0,0 @@
-Strategic Focus: What are the primary areas of focus in the strategic development plan for the technology firm, and why were they chosen?
-Revenue Growth: How does the company plan to achieve a 20% increase in revenue through expansion into emerging markets?
-Innovation Investment: What specific types of AI algorithms and cloud solutions is the company planning to develop with the allocated $100 million R&D investment?
-Partnerships and Acquisitions Strategy: Can you explain the criteria used for selecting startups for partnerships and acquisitions in the AI and IoT sectors?
-Risk Management: What are the key risks identified for the company, and what strategies are in place to mitigate these risks?
-Financial Projections: Based on the strategic initiatives outlined, what are the projected financial outcomes for the company over the next three years?
-Market Competition: How does the company plan to continuously analyze and adapt to competitive strategies in the technology sector?
-Regulatory Compliance: What measures will the company take to ensure adherence to global regulations, especially when expanding into new markets?
-Product Portfolio: How will the strategic partnerships and acquisitions enhance the company's product portfolio?
-Sustainability: What steps is the company taking to ensure the sustainability of its growth in the face of technological changes and market competition?

models/without_pronoun_cml_xgboost.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:933d1d5c5f83c30211dd9a497482c517a822df809c0498fed164de72bd7bf910
+size 1085795

models/without_pronoun_embedded_model.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:762240ca4040c68e44c403f16abce5683a0c4a005ec10f3dd0135a0e429a66c1
+size 1189196

models/without_pronoun_embedded_model.model.wv.vectors_ngrams.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cf06fe78185b373c97ee0616f599ce6b1aceb6445b8f666fac6cd4cd307fe46
+size 400000128

original_document_uuid_mapping.json

@@ -1,34 +1,35 @@
 {
-    "078-05-1126": "11e90148",
-    "1234567A": "c6652117",
-    "16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ": "be0eddce",
-    "18": "ee1fa38f",
-    "191280342": "53b52085",
-    "192.168.0.1": "2d66aefe",
-    "212": "81133b16",
-    "4095-2609-9393-4932": "1155d2a5",
-    "555-1234": "08aeb9f5",
-    "954567876544": "e1a23ada",
-    "Account": "3730179e",
-    "Bank": "5a60bcfa",
-    "David": "84e24eb8",
-    "Her": "67e5e61e",
-    "I": "9f2963af",
-    "IL150120690000003111111": "ab18e6bb",
-    "International": "11e27c41",
-    "Johnson": "0ea0e35c",
-    "Kate": "7ab7335c",
-    "Maine": "5d0a593d",
-    "My": "6d7a71e0",
-    "Number": "97008100",
-    "September": "b926f540",
-    "This": "3bbcaf02",
-    "crypto": "d8e7c7fc",
-    "microsoft.com": "1e63e774",
-    "my": "0a182475",
-    "please": "f4953628",
-    "status": "8ff83dc3",
-    "[email protected]": "f70cf334",
-    "wallet": "c627b841",
-    "you": "773b08cb"
+    "078-05-1126": "48f7c8a4",
+    "1234567A": "ab7ec0c3",
+    "16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ": "edf660df",
+    "18": "3f343449",
+    "191280342": "c263d176",
+    "192.168.0.1": "c83dd929",
+    "212": "a054c8c2",
+    "4095-2609-9393-4932": "bfd59a59",
+    "555-1234": "8fddc160",
+    "954567876544": "148fea84",
+    "Account": "3658044b",
+    "Bank": "49b9cffb",
+    "David": "97dc4202",
+    "IL150120690000003111111": "2f075e1d",
+    "IP": "c591bc5d",
+    "International": "c5462fed",
+    "Johnson": "7ce27ecb",
+    "Kate": "a18f3dda",
+    "Maine": "aaf4b006",
+    "September": "d615d819",
+    "account": "aa960526",
+    "bank": "4fd4e4c4",
+    "check": "e51c8e1c",
+    "crypto": "c7184a17",
+    "id": "f8380bf5",
+    "microsoft.com": "b6394fb9",
+    "phone": "c402f998",
+    "sent": "732237ac",
+    "social": "bcda6774",
+    "status": "f1b9c36f",
+    "[email protected]": "a295c5d0",
+    "valid": "08876c6f",
+    "wallet": "516361a1"
 }

utils_demo.py

@@ -0,0 +1,24 @@
+import uuid
+
+def process_tokens(tokens, inverse_uuid_map=None, uuid_map=None, embeddings_model=None, fhe_ner_detection=None, client=None):
+    """Processes tokens based on the provided parameters for either deanonymizing, anonymizing or default processing."""
+    processed_tokens = []
+    for token in tokens:
+        if not token.strip() or not re.match(r"\w+", token):  # Directly append non-word tokens or whitespace
+            processed_tokens.append(token)
+            continue
+        if inverse_uuid_map is not None:  # For deanonymizing response
+            processed_tokens.append(inverse_uuid_map.get(token, token))
+        elif uuid_map is not None and embeddings_model is not None and fhe_ner_detection is not None and client is not None:  # For FHEAnonymizer call
+            x = embeddings_model.wv[token][None]
+            prediction_proba = fhe_ner_detection.predict_proba(x)
+            probability = prediction_proba[0][1]
+            if probability >= 0.5:
+                tmp_uuid = uuid_map.get(token, str(uuid.uuid4())[:8])
+                processed_tokens.append(tmp_uuid)
+                uuid_map[token] = tmp_uuid
+            else:
+                processed_tokens.append(token)
+        else:
+            processed_tokens.append(token)
+    return ''.join(processed_tokens)