added imgur bucket

app.py

@@ -1,14 +1,23 @@
 from fastapi import FastAPI
 import uvicorn
+import faiss
 from sentence_transformers import SentenceTransformer
 
 model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+index = faiss.IndexFlatL2(model.get_sentence_embedding_dimension())   # build the index
+
+index.add(model.encode(['hello']))
 
 app = FastAPI()
 
+
+
 @app.post('/tts')
 async def transcribe(text: str):
-    embeddings = model.encode([text], vad_filter=True)
+    embeddings = model.encode([text])
+
+    # store the text to a file
+
     return embeddings[0]
 
 

db/__init__.py

@@ -0,0 +1,4 @@
+from dotenv import load_dotenv
+
+load_dotenv('./.env')
+

db/audio_db/is3/demo.py

@@ -0,0 +1,32 @@
+import asyncio
+
+
+async def demo():
+    import is3
+    
+    # creating a new bucket
+    bucket = is3.Bucket('my-is3-bucket')
+
+    # adding items to a bucket and uploading them
+    some_object = {'hello': b'world', ('foo', 'bar'): {'baz'}}
+    bucket.stage_obj(some_object, 'my-object')
+    bucket.stage_obj(['another', 'one'], 'my-other-object')
+
+    await bucket.commit()
+
+    # loading a bucket from disk
+    bucket = is3.Bucket.load('my-is3-bucket')
+
+    # retrieving items stored in a bucket
+    retrieved_object = await bucket.get_obj('my-object')
+    assert retrieved_object == {'hello': b'world', ('foo', 'bar'): {'baz'}}
+
+    # delete a specific item in a bucket
+    await bucket.delete_obj('my-object')
+
+    # delete an entire bucket and its contents
+    await bucket.delete()
+
+
+loop = asyncio.new_event_loop()
+loop.run_until_complete(demo())

db/audio_db/is3/is3.py

@@ -0,0 +1,146 @@
+import pickle
+import asyncio
+from pathlib import Path
+from pydantic import BaseModel
+
+from typing import Any
+from PIL import Image
+ObjectName = ObjectId = BucketName = str
+
+from .utils import image_to_object, object_to_image, write_compressed, read_compressed
+from .wrapper import ImgurClient as Imgur
+
+BUCKETS_FOLDER = Path(__file__).parent / 'buckets'
+EXTENSION = '.bkt'
+
+def filename(name: str):
+    return name + EXTENSION
+
+
+class UploadedObject(BaseModel):
+    """Represents an object that has been uploaded"""
+    name: ObjectName
+    obj_id: ObjectId
+    deletehash: str
+    cached_obj: Any = None
+
+    def __getstate__(self):
+        d = super().__getstate__()
+        d['__dict__']['cached_obj'] = None
+        d['__fields_set__'].discard('cached_obj')
+        return d
+
+    async def download(self) -> Any:
+        """Return the wrapped object."""
+        if self.cached_obj is not None:
+            return self.cached_obj
+
+        async with Imgur() as imgur:
+            img = await imgur.download_image(self.obj_id)
+
+        obj = self.cached_obj = image_to_object(img)
+        return obj
+    
+    async def delete(self) -> None:
+        """Delete the uploaded object"""
+        async with Imgur() as imgur:
+            await imgur.delete_image(self.deletehash)
+
+
+class StagedObject(BaseModel):
+    """Represents a bucket object that has been added to a bucket but not yet
+    uploaded"""
+    name: ObjectName
+    obj: Any
+
+    def image(self) -> Image.Image:
+        return object_to_image(self.obj)
+    
+    async def upload(self) -> UploadedObject:
+        """Upload the wrapped object and return an UploadedObject.
+        
+        The wrapped object is cached to the UploadObject so that a retrieval
+        during the same runtime does not need to download the object.
+        """
+        async with Imgur() as imgur:
+            oid, delete = await imgur.upload_image(self.image())
+        
+        return UploadedObject(
+            name=self.name,
+            obj_id=oid,
+            deletehash=delete,
+            cached_obj=self.obj
+        )
+
+
+class Bucket:
+    def __init__(self, name) -> None:
+        self.name = name
+        self.uploaded: dict[ObjectName, UploadedObject] = {}
+        self.pending: dict[ObjectName, StagedObject] = {}
+    
+    def __repr__(self) -> str:
+        n_pending = len(self.pending)
+        n_uploaded = len(self.uploaded)
+        return f'<Bucket {self.name} (pending: {n_pending}, uploaded: {n_uploaded})>'
+    
+    def _save(self):
+        """Pickle and dump the bucket to the buckets folder"""
+        fn = filename(self.name)
+        write_compressed(pickle.dumps(self), BUCKETS_FOLDER / fn)
+
+    def stage_obj(self, obj: Any, name: str) -> None:
+        self.pending[name] = StagedObject(obj=obj, name=name)
+
+        self._save()
+    
+    def unstage_obj(self, name: str) -> None:
+        del self.pending[name]
+        self._save()
+    
+    async def commit(self):
+        """Upload all staged objects"""
+        coros = [o.upload() for o in self.pending.values()]
+
+        # upload concurrently and filter out errors
+        results = await asyncio.gather(*coros, return_exceptions=True)
+        uploaded = [e for e in results if isinstance(e, UploadedObject)]
+        
+        # remove succesful uploads from pending
+        for o in uploaded:
+            del self.pending[o.name]
+
+        # track uploaded objects
+        self.uploaded.update({o.name: o for o in uploaded})
+
+        self._save()
+
+        # warn about unuploaded pending objects
+        if self.pending:
+            msg = (
+                f"{len(self.pending)} objects failed to upload:\n" +
+                '\n'.join(o.name for o in self.pending.values())
+            )
+            raise Warning(msg)
+    
+    async def get_obj(self, name: str) -> Any:
+        return await self.uploaded[name].download()
+
+    async def delete_obj(self, name: str) -> None:
+        """Remove the object with the given name from uploaded objects"""
+        if not (o := self.uploaded.pop(name, 0)):
+            raise ValueError(f'No obj with name {name} found in {self}')
+
+        await o.delete()
+        self._save()
+    
+    async def delete(self):
+        """Delete the bucket and all objects it holds"""
+        coros = [o.delete() for o in self.uploaded.values()]
+        await asyncio.gather(*coros)
+        (BUCKETS_FOLDER / filename(self.name)).unlink()
+
+    @classmethod
+    def load(cls, name: str) -> "Bucket":
+        fn = filename(name)
+        return pickle.loads(read_compressed(BUCKETS_FOLDER / fn))

db/audio_db/is3/utils.py

@@ -0,0 +1,108 @@
+import io
+import math
+import zlib
+import base64
+import pickle
+import numpy as np
+from PIL import Image
+
+from os import PathLike
+from typing import Any
+
+HEADER_SIZE = 4  # bytes
+
+
+def object_to_image(obj: Any) -> Image.Image:
+    """Take a object and convert it to an image.
+    
+    The object is first pickled to bytes, then the array is padded and reshaped
+    into a NxNx4 array and converted to an RGBA image
+
+    Example of 1d pickle array to image array:
+    [1,2,3,4,5,6,7] ->
+    [
+        [[0,0,0,7], [1,2,3,4]],
+        [[5,6,7,0], [0,0,0,0]]
+    ]
+    The array begins with a 4 byte header representing the length of the data.
+    Zeroes are added to the end the ensure that the number pixels is a suare number
+    """
+    data = compress(pickle.dumps(obj))
+    header = len(data).to_bytes(length=HEADER_SIZE, byteorder='big')
+    data = header + data
+
+    # divide the data into pixels, add an extra if data doesn't perfectly fit
+    whole_pixels, remainder = divmod(len(data), 4)
+    n_pixels = whole_pixels + 1 * (remainder != 0)
+
+    # ensure n_pixels is a square number
+    side_length = math.ceil(math.sqrt(n_pixels))
+    n_pixels = side_length ** 2
+    n_bytes = n_pixels * 4
+
+    # right pad the data with zeros so it can be shaped to (n,n,4)
+    data += b'\x00' * (n_bytes - len(data))
+    
+    # create (n,n,4) array from pickle data
+    data_arr = np.frombuffer(data, dtype=np.uint8)
+    img_arr = np.reshape(data_arr, (side_length, side_length, 4))
+
+    return Image.fromarray(img_arr)
+
+
+def image_to_object(image: Image.Image) -> Any:
+    """Take a PIL Image and unpickle it's data to an object
+    
+    Convert the image to an array, flatten to obtain serial bytes, then unpickle
+    these bytes.
+    """
+    data_arr = np.array(image).flatten()
+    data = data_arr.tobytes()
+
+    # number of bytes containing meaningful data
+    length = int.from_bytes(data[:HEADER_SIZE], 'big')
+
+    # slice off header
+    data = data[HEADER_SIZE:]
+    # slice off zero padding if any
+    data = data[:length]
+
+    return pickle.loads(decompress(data))
+
+
+def image_to_b64_string(img: Image.Image) -> str:
+    """Return a str representing the image as b64 encoded bytes"""
+    # save the image as PNG to a buffer
+    buffer = io.BytesIO()
+    img.save(buffer, 'png')
+    buffer.seek(0)
+
+    return base64.b64encode(buffer.read()).decode()
+
+
+def bytes_to_image(b: bytes) -> Image.Image:
+    """Create an Image using raw bytes"""
+    buffer = io.BytesIO(b)
+    buffer.seek(0)
+    
+    return Image.open(buffer)
+
+
+def compress(b: bytes, level=9) -> bytes:
+    """Compress the bytes using zlib"""
+    return zlib.compress(b, level)
+
+
+def decompress(b: bytes) -> bytes:
+    """Decompress the bytes using zlib"""
+    return zlib.decompress(b)
+
+
+def write_compressed(data: bytes, fp: PathLike) -> None:
+    with open(fp, 'wb') as f:
+        f.write(compress(data))
+
+
+def read_compressed(fp: PathLike) -> bytes:
+    with open(fp, 'rb') as f:
+        return decompress(f.read())

db/audio_db/is3/wrapper.py

@@ -0,0 +1,97 @@
+
+"""This module proivdes wrapper functionality for the imgur API"""
+
+import dotenv, os
+from PIL import Image
+import asyncio
+from aiohttp import ClientSession
+
+from typing import Optional, Union, Tuple
+
+from .utils import image_to_b64_string, bytes_to_image
+
+dotenv.load_dotenv()
+AUTH_HEADER = {'Authorization': f"Client-ID {os.getenv('IS3_CLIENT_ID')}"}
+API_ENDPOINTS = {
+    'upload': 'https://api.imgur.com/3/upload/',
+    'download': 'http://i.imgur.com/',
+    'info': 'https://api.imgur.com/3/image/',
+    'delete': 'https://api.imgur.com/3/image/',
+    'auth': f'https://api.imgur.com/oauth2/token'
+    }
+
+# get access and refresh token
+async def get_tokens():
+    session = ClientSession()
+    r = await session.request(
+            method='post',
+            url=API_ENDPOINTS['auth'],
+            headers=AUTH_HEADER,
+            data={
+                'refresh_token': os.getenv("IS3_REFRESH_TOKEN"),
+                'client_id': os.getenv("IS3_CLIENT_ID"),
+                'client_secret': os.getenv("IS3_CLIENT_SECRET"),
+                'grant_type': 'refresh_token',
+            }
+        )
+    r = await r.json()
+    return r['access_token'], r['refresh_token']
+
+ACCESS_TOKEN, REFRESH_TOKEN = asyncio.run(get_tokens())
+
+
+
+class ImgurClient:
+    """Class to interact with various API endpoints"""
+    def __init__(self, session: Optional[ClientSession] = None) -> None:
+        self._session = session or ClientSession()
+
+    async def __aenter__(self):
+        return self
+    
+    async def __aexit__(self, *err):
+        await self._session.close()
+
+    async def _request(self, method: str, url: str, *args, **kwargs) -> Union[dict, bytes]:
+        """Make a request with the specified method to the endpoint. All requests
+        should either return raw image data as bytes or other data as JSON"""
+        async with self._session.request(method, url, *args, **kwargs) as resp:
+            content_type = resp.content_type
+            if content_type == 'image/png':
+                return await resp.read()
+            elif content_type == 'application/json':
+                return (await resp.json())['data']
+            else:
+                raise RuntimeError(f'Unexpected response content-type "{content_type}"')
+
+    async def upload_image(self, img: Image.Image) -> Tuple[str, str]:
+        """Upload an image and return img id and deletehash"""
+        data = image_to_b64_string(img)
+        r = await self._request(
+            method='post',
+            url=API_ENDPOINTS['upload'],
+            headers=AUTH_HEADER, 
+            data={'image': data, 'type': 'base64'}
+        )
+        return r['id'], r['deletehash']
+    
+    async def download_image(self, image_id: str) -> Image.Image:
+        """Download the image and return the data as bytes."""
+        url = API_ENDPOINTS['download'] + image_id + '.png'
+        data = await self._request('get', url)
+
+        return bytes_to_image(data)
+        
+    async def delete_image(self, deletehash: str) -> None:
+        """Delete an image using a deletehash string"""
+        url = API_ENDPOINTS['delete'] + deletehash
+        await self._request('delete', url, headers=AUTH_HEADER)
+
+
+async def get_token():
+    im = ImgurClient()
+    await im.get_access_token()
+
+import asyncio
+loop = asyncio.get_event_loop()
+loop.run_until_complete(get_token())

db/create_db.py

@@ -0,0 +1,52 @@
+import os
+from pymilvus import (
+    utility,
+    FieldSchema,
+    CollectionSchema,
+    DataType,
+    Collection,
+)
+
+from db_connect import connect
+
+connect()
+
+#region creating collections
+### Create collections ###
+fields = [
+    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
+    FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=65535),
+    FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=384)
+]
+schema = CollectionSchema(fields, "Texts to generate audio for. "
+                                  "This collection cache the texts needed to generate audio. "
+                                  "We can then do offline generation for the audio file.")
+utility.drop_collection("Response")
+response_collection = Collection("Response", schema)
+index_params = {
+  "metric_type": "COSINE",
+  "index_type": "IVF_FLAT",
+  "params": {"nlist": 1024}
+}
+response_collection.create_index(field_name='embeddings', index_params=index_params)
+utility.index_building_progress("Response")
+
+
+fields = [
+    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
+    FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=65535),
+    FieldSchema(name="filename", dtype=DataType.VARCHAR, max_length=65535),
+    FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=384)
+]
+audio_schema = CollectionSchema(fields, "The text that corresponds to the audio file.")
+utility.drop_collection("AudioResponse")
+audio_response_collection = Collection("AudioResponse", audio_schema)
+audio_response_collection = Collection("AudioResponse")
+index_params = {
+  "metric_type": "COSINE",
+  "index_type": "IVF_FLAT",
+  "params": {"nlist": 1024}
+}
+audio_response_collection.create_index(field_name='embeddings', index_params=index_params)
+utility.index_building_progress("AudioResponse")
+#endregion

db/db_connect.py

@@ -0,0 +1,16 @@
+import os
+from pymilvus import connections
+
+
+def connect():
+    # Initialize a MilvusClient instance
+    # Replace uri and API key with your own
+    connections.connect(
+        alias=os.getenv('MILVUS_DEFAULT'),
+        uri=os.getenv("MILVUS_CLUSTER_ENDPOINT"), # Cluster endpoint obtained from the console
+        token=os.getenv("MILVUS_TOKEN")  # API key or a colon-separated cluster username and password
+    )
+
+
+def disconnect():
+    connections.disconnect("MILVUS_DEFAULT")

db/generate_audio.py

@@ -0,0 +1,45 @@
+from sentence_transformers import SentenceTransformer
+from pymilvus import Collection
+import asyncio
+
+from db_connect import connect
+
+# for audio storage
+from audio_db.is3 import is3
+
+connect()
+
+async def generate_audio():
+    response = Collection("Response")
+    audio_response = Collection("AudioResponse")
+
+    data = []
+
+    response_iterator = response.query_iterator(limit=1, output_fields=['text', 'embeddings'])
+    while True:
+        res = response_iterator.next()
+        if len(res) == 0:
+            print("query iteration finished, close")
+            # close the iterator
+            response_iterator.close()
+            break
+
+        bucket = is3.Bucket(str(res[0]['id']))
+
+        # generate audio
+        audio_bytes = open('445766006129375465.wav', 'rb').read()
+
+        # store the audio
+        bucket.stage_obj(audio_bytes, 'audio')
+        await bucket.commit()
+
+        # save the audio record to AudioResponse
+        data.append([res[0]['text'], str(res[0]['id']), res[0]['embeddings']])
+
+    audio_response.insert(list(zip(*data)))
+    audio_response.flush()
+
+
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(generate_audio())

db/load_db.py

@@ -0,0 +1,27 @@
+from sentence_transformers import SentenceTransformer
+# Connect using a MilvusClient object
+from pymilvus import Collection
+
+from db_connect import connect
+
+#
+with open('responses.txt', 'r') as file:
+  content = file.read()
+  contents = content.split('\n\n')
+
+model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+embeddings = model.encode(contents)
+
+data = [
+    contents,
+    embeddings
+]
+
+connect()
+
+collection = Collection("Response")
+collection.insert(data)
+collection.flush()
+
+
+

db/responses.txt

@@ -0,0 +1,87 @@
+Our check-in time is at 3 PM, and check-out is at 11 AM. Let us know if you need any adjustments.
+
+Yes, we offer complimentary Wi-Fi throughout the hotel for all our guests.
+
+Currently, we offer a complimentary continental breakfast from 7 AM to 10 AM in the lobby area.
+
+Yes, we provide free parking for our guests. There's no need for reservations for parking spaces.
+
+Absolutely! Red Roof Inn is pet-friendly, and your furry friends can stay for free. Please be aware of our pet policy for any size or breed restrictions.
+
+We have a fitness center open from 6 AM to 10 PM. However, we do not have a pool at this location.
+
+Early check-in and late check-out requests are subject to availability. Please contact the front desk to arrange it.
+
+You can book directly through our website, call us, or use any major hotel booking platform. We're happy to assist with your reservation!
+
+Yes, we have various discounts including for AAA members, seniors, government employees, and military personnel. Please inquire for more details.
+
+Certainly! There are several dining options and shops within a short distance. Our front desk can provide recommendations and directions.
+
+Good afternoon! Welcome to our hotel. Our Wi-Fi is complimentary and available throughout the hotel. For breakfast, we offer a free continental breakfast in the lobby from 7 AM to 10 AM.
+
+Yes, we provide free parking for our guests. There's no need to reserve a space; you can park anywhere available.
+
+Absolutely! We're a pet-friendly hotel, so your furry friends are welcome to stay for free. There are some size and breed restrictions, so please let us know if you have any specific concerns.
+
+Of course! There are several excellent restaurants within a short walking distance. I can provide you with a list and directions if you like.
+
+You're welcome! If you have any more questions or need assistance during your stay, please don't hesitate to ask. Enjoy your stay!
+
+Thank you for bringing this to our attention. We'll investigate and take the necessary action immediately.
+
+I apologize for the delay. I'll check with our staff and ensure your luggage is delivered promptly.
+
+We're sorry for the inconvenience. It's being repaired, and we're happy to assist you with the stairs or alternative routes.
+
+I understand. Let's find you a quieter room further from the elevator.
+
+I apologize for that oversight. We'll have it stocked immediately.
+
+We're sorry for the discomfort. We'll have someone look at the thermostat or offer a different room.
+
+Thank you for notifying us. We'll make sure it's cleaned right away for your safety and comfort.
+
+We apologize for the inconvenience. We'll work on getting them repaired or replaced as soon as possible.
+
+Let's review your bill together. We'll correct any discrepancies immediately.
+
+That's unacceptable, and I apologize. We'll resolve this issue at once or offer you a different room.
+
+We're sorry to hear that. We'll send someone to fix it or offer you a room with proper curtains.
+
+Thank you for your patience while I look into this.
+
+Just a brief moment, please.
+
+I'll be right with you, please hold on.
+
+Let me check that for you, please stay on the line.
+
+I'll be back with you shortly, please bear with me.
+
+Please allow me a moment to address your request.
+
+I'll need a moment to find the answer, please hold tight.
+
+I'll work on it, please hold the line.
+
+I'll look into that right now, please stay on the call.
+
+Just a quick pause while I verify the details.
+
+I'll find the information, please wait.
+
+I'm investigating that for you, please give me a second.
+
+I'm going to check, please hold on for a moment.
+
+I'll be with you shortly, thank you for your patience.
+
+I'm actively working on your request, please stay on hold.
+
+Please allow me a moment to research and provide an answer.
+
+I'm gathering the necessary details, please bear with me.
+
+I'll get back to you shortly, please remain on the line.

requirements.txt

@@ -1,4 +1,6 @@
 sentence_transformers==2.2.2
 fastapi==0.104.1
 uvicorn==0.24.0.post1
-python-multipart==0.0.6
+python-multipart==0.0.6
+faiss-cpu==1.7.4
+pymilvus==2.3.3

responses.txt

@@ -1,29 +0,0 @@
-"Our check-in time is at 3 PM, and check-out is at 11 AM. Let us know if you need any adjustments."
-
-"Yes, we offer complimentary Wi-Fi throughout the hotel for all our guests."
-
-"Currently, we offer a complimentary continental breakfast from 7 AM to 10 AM in the lobby area."
-
-"Yes, we provide free parking for our guests. There's no need for reservations for parking spaces."
-
-"Absolutely! Red Roof Inn is pet-friendly, and your furry friends can stay for free. Please be aware of our pet policy for any size or breed restrictions."
-
-"We have a fitness center open from 6 AM to 10 PM. However, we do not have a pool at this location."
-
-"Early check-in and late check-out requests are subject to availability. Please contact the front desk to arrange it."
-
-"You can book directly through our website, call us, or use any major hotel booking platform. We're happy to assist with your reservation!"
-
-"Yes, we have various discounts including for AAA members, seniors, government employees, and military personnel. Please inquire for more details."
-
-"Certainly! There are several dining options and shops within a short distance. Our front desk can provide recommendations and directions."
-
-"Good afternoon! Welcome to our hotel. Our Wi-Fi is complimentary and available throughout the hotel. For breakfast, we offer a free continental breakfast in the lobby from 7 AM to 10 AM."
-
-"Yes, we provide free parking for our guests. There's no need to reserve a space; you can park anywhere available."
-
-"Absolutely! We're a pet-friendly hotel, so your furry friends are welcome to stay for free. There are some size and breed restrictions, so please let us know if you have any specific concerns."
-
-"Of course! There are several excellent restaurants within a short walking distance. I can provide you with a list and directions if you like."
-
-"You're welcome! If you have any more questions or need assistance during your stay, please don't hesitate to ask. Enjoy your stay!"