AnswerToSpeech

haystack/nodes/audio/answer_to_speech.py

@@ -0,0 +1,60 @@
+import logging
+from typing import Union, List, Dict, Any, Tuple
+
+import os
+import hashlib
+from pathlib import Path
+
+from espnet2.bin.tts_inference import Text2Speech
+import soundfile as sf
+
+from haystack.nodes import BaseComponent
+from haystack.schema import Answer, AudioAnswer, GeneratedAudioAnswer
+
+
+class AnswerToSpeech(BaseComponent):
+
+    outgoing_edges = 1
+
+    def __init__(
+        self,
+        model_name_or_path: Union[str, Path] = "espnet/kan-bayashi_ljspeech_vits",
+        generated_audio_path: Path = Path(__file__).parent / "generated_audio_answers",
+    ):
+        super().__init__()
+        self.model = Text2Speech.from_pretrained(model_name_or_path)
+        self.generated_audio_path = generated_audio_path
+
+        if not os.path.exists(self.generated_audio_path):
+            os.mkdir(self.generated_audio_path)
+
+    def text_to_speech(self, text: str) -> Any:
+        filename = hashlib.md5(text.encode("utf-8")).hexdigest()
+        path = self.generated_audio_path / f"{filename}.wav"
+
+        # Duplicate answers might be in the list, in this case we save time by not regenerating.
+        if not os.path.exists(path):
+            output = self.model(text)["wav"]
+            sf.write(path, output.numpy(), self.model.fs, "PCM_16")
+
+        return path
+
+    def run(self, answers: List[Answer]) -> Tuple[Dict[str, AudioAnswer], str]:
+
+        audio_answers = []
+        for answer in answers:
+
+            logging.info(f"Processing answer '{answer.answer}' and its context...")
+            answer_audio = self.text_to_speech(answer.answer)
+            context_audio = self.text_to_speech(answer.context)
+
+            audio_answer = GeneratedAudioAnswer.from_text_answer(
+                answer_object=answer, generated_audio_answer=answer_audio, generated_audio_context=context_audio
+            )
+            audio_answer.type = "generative"
+            audio_answers.append(audio_answer)
+
+        return {"answers": audio_answers}, "output_1"
+
+    def run_batch(self, answers: List[Answer]) -> Tuple[Dict[str, AudioAnswer], str]:
+        return self.run(answers)

haystack/schema.py

@@ -327,6 +327,42 @@ def from_json(cls, data):
         return cls.from_dict(data)
 
 
+@dataclass
+class AudioAnswer(Answer):
+    answer: Path
+    context: Optional[Path] = None
+    offsets_in_document: Optional[Any] = None
+    offsets_in_context: Optional[Any] = None
+
+    def __str__(self):
+        return f"<AudioAnswer: answer='{self.answer}', score={self.score}, context='{self.context}'>"
+
+    def __repr__(self):
+        return f"<AudioAnswer {asdict(self)}>"
+
+
+@dataclass
+class GeneratedAudioAnswer(AudioAnswer):
+    type: str = "text-to-speech"
+    answer_transcript: Optional[str] = None
+    context_transcript: Optional[str] = None
+
+    @classmethod
+    def from_text_answer(
+        cls, answer_object: Answer, generated_audio_answer: Any, generated_audio_context: Optional[Any] = None
+    ):
+        answer_dict = answer_object.to_dict()
+        answer_dict = {key: value for key, value in answer_dict.items() if value}
+
+        answer_dict["answer_transcript"] = answer_dict["answer"]
+        answer_dict["context_transcript"] = answer_dict["context"]
+
+        answer_dict["answer"] = generated_audio_answer
+        answer_dict["context"] = generated_audio_context
+
+        return cls(**answer_dict)
+
+
 @dataclass
 class Label:
     id: str

setup.cfg

@@ -150,6 +150,12 @@ docstores =
     farm-haystack[faiss,milvus,weaviate,graphdb,pinecone]
 docstores-gpu =
     farm-haystack[faiss-gpu,milvus,weaviate,graphdb,pinecone]
+
+audio = 
+    espnet
+    espnet-model-zoo
+beir = 
+    beir; platform_system != 'Windows'
 crawler = 
     selenium
     webdriver-manager
@@ -172,10 +178,9 @@ ray =
     ray>=1.9.1,<2; platform_system != 'Windows'
     ray>=1.9.1,<2,!=1.12.0; platform_system == 'Windows'  # Avoid 1.12.0 due to https://github.com/ray-project/ray/issues/24169 (fails on windows)
     aiorwlock>=1.3.0,<2
+
 colab = 
     grpcio==1.43.0
-beir = 
-    beir; platform_system != 'Windows'
 dev = 
     # Type check
     mypy