INNER CODE UNIT · Python

process_metadata

junruxiong/IncarnaMind · docs2db.py:174

def process_metadata(doc: List[Document]):
    """Processes and updates the metadata for a list of Document objects.

    Args:
        doc (list): List of Document objects.
    """
    # get file name and remove extension
    file_name_with_extension = os.path.basename(doc[0].metadata["source"])
    file_name, _ = os.path.splitext(file_name_with_extension)

    for _, item in enumerate(doc):
        for key, value in item.metadata.items():
            if isinstance(value, list):
                item.metadata[key] = str(value)
        item.metadata["page_content"] = item.page_content
        item.metadata["page_content_md5"] = string2md5(item.page_content)
        item.metadata["source_md5"] = string2md5(item.metadata["source"])
        item.page_content = f"{file_name}\n{item.page_content}"

View source record →