Add vector data

app.py

@@ -0,0 +1,61 @@
+from langchain import HuggingFacePipeline
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import BSHTMLLoader, DirectoryLoader
+from langchain.embeddings import HuggingFaceEmbeddings
+from transformers import AutoTokenizer
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import Chroma
+
+
+!git clone https://github.com/TheMITTech/shakespeare
+
+from glob import glob
+
+files = glob("./shakespeare/**/*.html")
+
+import shutil
+import os
+
+os.mkdir('./data')
+
+destination_folder = './data/'
+
+for html_file in files:
+      shutil.move(html_file, destination_folder + html_file.split("/")[-1])
+bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
+
+data = bshtml_dir_loader.load()
+
+bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
+
+text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
+                                                   chunk_size=100,
+                                                   chunk_overlap=0,
+                                                   separator="\n")
+
+documents = text_splitter.split_documents(data)
+
+embeddings = HuggingFaceEmbeddings()
+
+
+
+persist_directory = "vector_db"
+
+vectordb = Chroma.from_documents(documents=documents, embedding=embeddings,
+                                 persist_directory=persist_directory)
+
+llm = HuggingFacePipeline.from_model_id(
+        model_id="bigscience/bloomz-1b7",
+        task="text-generation",
+        model_kwargs={"temperature" : 0, "max_length" : 500})
+
+doc_retriever = vectordb.as_retriever()
+
+
+shakespeare_qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=doc_retriever)
+
+def query(query):
+    shakespeare_qa.run(query)
+
+iface = gr.Interface(fn=query, inputs="text", outputs="text")
+iface.launch()

requirements.txt

@@ -0,0 +1,7 @@
+langchain
+beautifulsoup4
+transformers
+huggingface-hub
+sentence_transformers
+chromadb
+