Update functions.py

functions.py

@@ -23,7 +23,7 @@ from pyvis.network import Network
 import torch
 from langchain.docstore.document import Document
 from langchain.embeddings import HuggingFaceEmbeddings,HuggingFaceInstructEmbeddings
-from langchain.vectorstores import Pinecone
+from langchain.vectorstores import FAISS
 from langchain.chains.qa_with_sources import load_qa_with_sources_chain
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.llms import OpenAI
@@ -43,8 +43,6 @@ time_str = time.strftime("%d%m%Y-%H%M%S")
 HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; 
 margin-bottom: 2.5rem">{}</div> """
 
-index_id = "earnings-embeddings"
-
 #Stuff Chain Type Prompt template
 output_parser = RegexParser(
     regex=r"(.*?)\nScore: (.*)",
@@ -125,25 +123,17 @@ def load_asr_model(asr_model_name):
     return asr_model
 
 @st.experimental_singleton(suppress_st_warning=True)
-def process_corpus(corpus, _tok, title, _embeddings, chunk_size=200, overlap=50):
+def process_corpus(corpus, _tokenizer, title, embedding_model, chunk_size=200, overlap=50):
 
     '''Process text for Semantic Search'''
     
-    pinecone.init(api_key=OPEN_AI_KEY, environment="us-west1-gcp")
-
-    tokenizer = _tok
-    text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(tokenizer,chunk_size=chunk_size,chunk_overlap=overlap,separator='. ')
+    text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(tokenizer,chunk_size=chunk_size,chunk_overlap=overlap)
 
     texts = text_splitter.split_text(corpus)
 
-    docsearch = Pinecone.from_texts(
-        texts,
-        _embeddings,
-        index_name = "earnings-embeddings",
-        namespace = f'{title}-earnings',
-        metadatas = [
-        {'source':i} for i in range(len(texts))]
-    )
+    embeddings = gen_embeddings(embedding_model)
+
+    docsearch = FAISS.from_texts(texts, embeddings)
 
     return docsearch
 
@@ -165,17 +155,15 @@ def gen_embeddings(embedding_model):
     return embeddings
     
 @st.experimental_memo(suppress_st_warning=True)
-def embed_text(query,corpus,title,embedding_model,_emb_tok,chain_type='stuff'):
+def embed_text(query,corpus,title,embedding_model,_emb_tok,_chain_type='Normal'):
     
     '''Embed text and generate semantic search scores'''
 
     title = title.split()[0].lower()
-    
-    embeddings = gen_embeddings(embedding_model)
         
-    docsearch = process_corpus(corpus,_emb_tok,title, embeddings)
+    docsearch = process_corpus(corpus,emb_tok,title, embedding_model)
         
-    docs = docsearch.similarity_search_with_score(query, k=3, namespace = f'{title}-earnings')
+    docs = docsearch.similarity_search_with_score(query, k=3)
 
     print(docs)