修复历史记录中文编码

modules/config.py

@@ -43,11 +43,11 @@ hide_history_when_not_logged_in = config.get("hide_history_when_not_logged_in",
 
 if os.path.exists("api_key.txt"):
     logging.info("检测到api_key.txt文件，正在进行迁移...")
-    with open("api_key.txt", "r") as f:
+    with open("api_key.txt", "r", encoding="utf-8") as f:
         config["openai_api_key"] = f.read().strip()
     os.rename("api_key.txt", "api_key(deprecated).txt")
     with open("config.json", "w", encoding='utf-8') as f:
-        json.dump(config, f, indent=4)
+        json.dump(config, f, indent=4, ensure_ascii=False)
 
 if os.path.exists("auth.json"):
     logging.info("检测到auth.json文件，正在进行迁移...")
@@ -63,7 +63,7 @@ if os.path.exists("auth.json"):
     config["users"] = auth_list
     os.rename("auth.json", "auth(deprecated).json")
     with open("config.json", "w", encoding='utf-8') as f:
-        json.dump(config, f, indent=4)
+        json.dump(config, f, indent=4, ensure_ascii=False)
 
 ## 处理docker if we are running in Docker
 dockerflag = config.get("dockerflag", False)

modules/index_func.py

@@ -16,7 +16,7 @@ def get_index_name(file_src):
 
     md5_hash = hashlib.md5()
     for file_path in file_paths:
-        with open(file_path, "rb") as f:
+        with open(file_path, "rb", encoding="utf-8") as f:
             while chunk := f.read(8192):
                 md5_hash.update(chunk)
 
@@ -47,7 +47,7 @@ def get_documents(file_src):
                     pdftext = parse_pdf(filepath, two_column).text
                 except:
                     pdftext = ""
-                    with open(filepath, "rb") as pdfFileObj:
+                    with open(filepath, "rb", encoding="utf-8") as pdfFileObj:
                         pdfReader = PyPDF2.PdfReader(pdfFileObj)
                         for page in tqdm(pdfReader.pages):
                             pdftext += page.extract_text()

modules/models/base_model.py

@@ -641,7 +641,7 @@ class BaseLLMModel:
                 history_file_path = os.path.join(HISTORY_DIR, user_name, filename)
             else:
                 history_file_path = filename
-            with open(history_file_path, "r") as f:
+            with open(history_file_path, "r", encoding="utf-8") as f:
                 json_s = json.load(f)
             try:
                 if type(json_s["history"][0]) == str:

modules/models/models.py

@@ -338,7 +338,7 @@ class LLaMA_Client(BaseLLMModel):
             pipeline_args = InferencerArguments(
                 local_rank=0, random_seed=1, deepspeed='configs/ds_config_chatbot.json', mixed_precision='bf16')
 
-            with open(pipeline_args.deepspeed, "r") as f:
+            with open(pipeline_args.deepspeed, "r", encoding="utf-8") as f:
                 ds_config = json.load(f)
             LLAMA_MODEL = AutoModel.get_model(
                 model_args,
@@ -623,7 +623,7 @@ def get_model(
 
 
 if __name__ == "__main__":
-    with open("config.json", "r") as f:
+    with open("config.json", "r", encoding="utf-8") as f:
         openai_api_key = cjson.load(f)["openai_api_key"]
     # set logging level to debug
     logging.basicConfig(level=logging.DEBUG)

modules/utils.py

@@ -256,8 +256,8 @@ def save_file(filename, system, history, chatbot, user_name):
             history_file_path = filename
         else:
             history_file_path = os.path.join(HISTORY_DIR, user_name, filename)
-        with open(history_file_path, "w") as f:
-            json.dump(json_s, f)
+        with open(history_file_path, "w", encoding='utf-8') as f:
+            json.dump(json_s, f, ensure_ascii=False)
     elif filename.endswith(".md"):
         md_s = f"system: \n- {system} \n"
         for data in history:
@@ -563,7 +563,7 @@ def toggle_like_btn_visibility(selected_model_name):
 def new_auto_history_filename(dirname):
     latest_file = get_latest_filepath(dirname)
     if latest_file:
-        with open(os.path.join(dirname, latest_file), 'r') as f:
+        with open(os.path.join(dirname, latest_file), 'r', encoding="utf-8") as f:
             if len(f.read()) == 0:
                 return latest_file
     now = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')