with voice clone

.gitignore

@@ -0,0 +1,165 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+*.wav
+flagged/

app.py

@@ -0,0 +1,85 @@
+import gradio as gr
+import os
+from transformers import pipeline, AutoModelForQuestionAnswering, AutoTokenizer
+from datasets import load_dataset
+import torch
+import soundfile as sf
+from pdfminer.high_level import extract_text
+from llama_cpp import Llama
+
+
+# Check if MPS is available and set the device
+if torch.backends.mps.is_available():
+    device = torch.device("mps")
+    print("Using MPS device")
+else:
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"MPS not available, using {device}")
+def toText(audio):
+    asr = pipeline(
+    "automatic-speech-recognition",
+    model="openai/whisper-tiny.en",
+    chunk_length_s=30,
+    device=device,
+    )
+    question = asr(audio, batch_size=8)["text"]
+    return question
+
+
+def extract_answer(question, text):
+    # Load the LLaMA model
+    model_path="/Users/chandima/.cache/lm-studio/models/lmstudio-community/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q3_K_L.gguf"
+    # Load the LLaMA model with MPS acceleration
+    llm = Llama(
+        model_path=model_path,
+        n_gpu_layers=-1,  # Use all available layers for GPU acceleration
+        n_ctx=2048,  # Adjust context size as needed
+        verbose=True,  # Optional: for debugging
+        use_mlock=True,  # Optional: for better memory management
+        n_threads=6,  # Adjust based on your CPU
+        use_mmap=True,  # Optional: for faster loading
+    )
+
+    # Use LLaMA to extract skills
+    prompt = f"""
+    Answer the question based on the Resume.
+
+    Question:
+    {question}:
+
+    Resume:
+    {text}
+
+    Answer:
+    """
+    
+    response = llm(prompt, max_tokens=800, stop=["Human:", "\n\n"])
+    answer = response['choices'][0]['text'].strip()
+    print(answer)
+    return answer
+
+def toAudio(text):
+    synthesiser = pipeline("text-to-speech", "microsoft/speecht5_tts", device=device)
+    embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+    speaker_embedding = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
+    speech = synthesiser(text, forward_params={"speaker_embeddings": speaker_embedding})
+    return speech
+
+def clone(audio, file):
+    question = toText(audio=audio)
+    text = extract_text(file.name)
+    res = extract_answer(question, text)
+    print(res)
+    speech = toAudio(res) 
+    sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])
+    return "./speech.wav" 
+
+iface = gr.Interface(fn=clone, 
+                     inputs=[gr.Audio(type='filepath', label='Voice reference audio file'), gr.File(label="Resume")], 
+                     outputs=gr.Audio(label='Says'),
+                     title='Voice Clone',
+                     description="""
+                     whisper
+                     """,
+                     theme = gr.themes.Base(primary_hue="teal",secondary_hue="teal",neutral_hue="slate"))
+iface.launch()

requirements.txt

@@ -0,0 +1,169 @@
+absl-py==2.1.0
+aiofiles==23.2.1
+aiohappyeyeballs==2.4.3
+aiohttp==3.10.8
+aiosignal==1.3.1
+annotated-types==0.7.0
+anyascii==0.3.2
+anyio==4.6.0
+async-timeout==4.0.3
+attrs==24.2.0
+audioread==3.0.1
+babel==2.16.0
+bangla==0.0.2
+blinker==1.8.2
+blis==0.7.11
+bnnumerizer==0.0.2
+bnunicodenormalizer==0.1.7
+catalogue==2.0.10
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.19.0
+confection==0.1.5
+contourpy==1.2.1
+coqpit==0.0.17
+cryptography==43.0.1
+cycler==0.12.1
+cymem==2.0.8
+Cython==3.0.11
+datasets==3.0.1
+dateparser==1.1.8
+decorator==5.1.1
+dill==0.3.8
+diskcache==5.6.3
+docopt==0.6.2
+einops==0.8.0
+encodec==0.1.1
+exceptiongroup==1.2.2
+fastapi==0.115.0
+ffmpy==0.4.0
+filelock==3.16.1
+Flask==3.0.3
+fonttools==4.54.1
+frozenlist==1.4.1
+fsspec==2024.6.1
+g2pkk==0.1.2
+gradio==4.44.1
+gradio_client==1.3.0
+grpcio==1.66.2
+gruut==2.2.3
+gruut-ipa==0.13.0
+gruut_lang_de==2.0.1
+gruut_lang_en==2.0.1
+gruut_lang_es==2.0.1
+gruut_lang_fr==2.0.2
+h11==0.14.0
+hangul-romanize==0.1.0
+httpcore==1.0.5
+httpx==0.27.2
+huggingface-hub==0.25.1
+idna==3.10
+importlib_resources==6.4.5
+inflect==7.4.0
+itsdangerous==2.2.0
+jamo==0.4.1
+jieba==0.42.1
+Jinja2==3.1.4
+joblib==1.4.2
+jsonlines==1.2.0
+kiwisolver==1.4.7
+langcodes==3.4.1
+language_data==1.2.0
+lazy_loader==0.4
+librosa==0.10.0
+llama_cpp_python==0.3.1
+llvmlite==0.43.0
+marisa-trie==1.2.0
+Markdown==3.7
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.8.4
+mdurl==0.1.2
+more-itertools==10.5.0
+mpmath==1.3.0
+msgpack==1.1.0
+multidict==6.1.0
+multiprocess==0.70.16
+murmurhash==1.0.10
+networkx==2.8.8
+nltk==3.9.1
+num2words==0.5.13
+numba==0.60.0
+numpy==1.22.0
+orjson==3.10.7
+packaging==24.1
+pandas==1.5.3
+pdfminer.six==20240706
+pillow==10.4.0
+platformdirs==4.3.6
+pooch==1.8.2
+preshed==3.0.9
+protobuf==5.28.2
+psutil==6.0.0
+pyarrow==17.0.0
+pycparser==2.22
+pydantic==2.9.2
+pydantic_core==2.23.4
+pydub==0.25.1
+Pygments==2.18.0
+pynndescent==0.5.13
+pyparsing==3.1.4
+pypinyin==0.53.0
+pysbd==0.3.4
+python-crfsuite==0.9.10
+python-dateutil==2.9.0.post0
+python-multipart==0.0.12
+pytz==2024.2
+PyYAML==6.0.2
+regex==2024.9.11
+requests==2.32.3
+rich==13.8.1
+ruff==0.6.8
+safetensors==0.4.5
+scikit-learn==1.5.2
+scipy==1.11.4
+semantic-version==2.10.0
+sentencepiece==0.2.0
+shellingham==1.5.4
+six==1.16.0
+smart-open==7.0.4
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.5.0.post1
+spacy==3.7.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+srsly==2.4.8
+starlette==0.38.6
+SudachiDict-core==20240716
+SudachiPy==0.6.8
+sympy==1.13.3
+tensorboard==2.18.0
+tensorboard-data-server==0.7.2
+thinc==8.2.5
+threadpoolctl==3.5.0
+tokenizers==0.20.0
+tomlkit==0.12.0
+torch==2.4.1
+torchaudio==2.4.1
+tqdm==4.66.5
+trainer==0.0.36
+transformers==4.45.1
+TTS==0.22.0
+typeguard==4.3.0
+typer==0.12.5
+typing_extensions==4.12.2
+tzlocal==5.2
+umap-learn==0.5.6
+Unidecode==1.3.8
+urllib3==2.2.3
+uvicorn==0.31.0
+wasabi==1.1.3
+weasel==0.4.1
+websockets==12.0
+Werkzeug==3.0.4
+wrapt==1.16.0
+xxhash==3.5.0
+yarl==1.13.1

tts.py

@@ -0,0 +1,14 @@
+from transformers import pipeline
+from datasets import load_dataset
+import soundfile as sf
+import torch
+
+synthesiser = pipeline("text-to-speech", "microsoft/speecht5_tts")
+
+embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+speaker_embedding = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
+# You can replace this embedding with your own as well.
+
+speech = synthesiser("Hello, my dog is cooler than you!", forward_params={"speaker_embeddings": speaker_embedding})
+
+sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])

whisper.py

@@ -0,0 +1,20 @@
+import torch
+from transformers import pipeline
+from datasets import load_dataset
+
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+pipe = pipeline(
+  "automatic-speech-recognition",
+  model="openai/whisper-tiny.en",
+  chunk_length_s=30,
+  device=device,
+)
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+sample = ds[0]["audio"]
+
+prediction = pipe(sample.copy(), batch_size=8)["text"]
+
+# we can also return timestamps for the predictions
+prediction = pipe(sample.copy(), batch_size=8, return_timestamps=True)["chunks"]