# # Copyright 2024 The InfiniFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Reference: - [graphrag](https://github.com/microsoft/graphrag) """ import html import re from collections.abc import Callable from typing import Any ErrorHandlerFn = Callable[[BaseException | None, str | None, dict | None], None] def perform_variable_replacements( input: str, history: list[dict]=[], variables: dict | None ={} ) -> str: """Perform variable replacements on the input string and in a chat log.""" result = input def replace_all(input: str) -> str: result = input if variables: for entry in variables: result = result.replace(f"{{{entry}}}", variables[entry]) return result result = replace_all(result) for i in range(len(history)): entry = history[i] if entry.get("role") == "system": history[i]["content"] = replace_all(entry.get("content") or "") return result def clean_str(input: Any) -> str: """Clean an input string by removing HTML escapes, control characters, and other unwanted characters.""" # If we get non-string input, just give it back if not isinstance(input, str): return input result = html.unescape(input.strip()) # https://stackoverflow.com/questions/4324790/removing-control-characters-from-a-string-in-python return re.sub(r"[\"\x00-\x1f\x7f-\x9f]", "", result) def dict_has_keys_with_types( data: dict, expected_fields: list[tuple[str, type]] ) -> bool: """Return True if the given dictionary has the given keys with the given types.""" for field, field_type in expected_fields: if field not in data: return False value = data[field] if not isinstance(value, field_type): return False return True