Web Summarize Chain Of Density

# API 키를 환경변수로 관리하기 위한 설정 파일
from dotenv import load_dotenv

# API 키 정보 로드
load_dotenv()

Chain of Density: https://arxiv.org/pdf/2309.04269.pdf

import time
import textwrap

from langchain import hub
from langchain_openai import ChatOpenAI
from langchain.output_parsers.json import SimpleJsonOutputParser
from langchain.document_loaders import WebBaseLoader
from langchain.schema.runnable import RunnablePassthrough

# Load some data to summarize
loader = WebBaseLoader("https://teddylee777.github.io/data-science/optuna/")
docs = loader.load()
content = docs[0].page_content

# Get this prompt template
prompt = hub.pull("lawwu/chain_of_density")

# The chat model output is a JSON list of dicts, with SimpleJsonOutputParser
# we can convert it o a dict, and it suppors streaming.
json_parser = SimpleJsonOutputParser()

chain = (
    {"ARTICLE": RunnablePassthrough()}
    | prompt
    | ChatOpenAI(model="gpt-3.5-turbo-16k", temperature=0.1)
    | json_parser
)

chain.invoke(content)

from langchain import hub
from langchain_openai import ChatOpenAI
from langchain_core.output_parsers import StrOutputParser, JsonOutputParser
from langchain.document_loaders import WebBaseLoader
from langchain_core.prompts import ChatPromptTemplate
from langchain.callbacks.base import BaseCallbackHandler
import json


# Load some data to summarize
loader = WebBaseLoader(
    "https://www.aitimes.com/news/articleView.html?idxno=131777")
docs = loader.load()
content = docs[0].page_content
# Load the prompt
# prompt = hub.pull("langchain-ai/chain-of-density:4f55305e")


class StreamCallback(BaseCallbackHandler):
    def on_llm_new_token(self, token, **kwargs):
        print(token, end="", flush=True)


prompt = ChatPromptTemplate.from_template(
    """Article: {ARTICLE}
You will generate increasingly concise, entity-dense summaries of the above article. 

Repeat the following 2 steps 5 times. 

Step 1. Identify 1-3 informative entities (";" delimited) from the article which are missing from the previously generated summary. 
Step 2. Write a new, denser summary of identical length which covers every entity and detail from the previous summary plus the missing entities. 

A missing entity is:
- relevant to the main story, 
- specific yet concise (50 words or fewer), 
- novel (not in the previous summary), 
- faithful (present in the article), 
- anywhere (can be located anywhere in the article).

Guidelines:

- The first summary should be long (8-10 sentences, ~200 words) yet highly non-specific, containing little information beyond the entities marked as missing. Use overly verbose language and fillers (e.g., "this article discusses") to reach ~200 words.
- Make every word count: rewrite the previous summary to improve flow and make space for additional entities.
- Make space with fusion, compression, and removal of uninformative phrases like "the article discusses".
- The summaries should become highly dense and concise yet self-contained, i.e., easily understood without the article. 
- Missing entities can appear anywhere in the new summary.
- Never drop entities from the previous summary. If space cannot be made, add fewer new entities. 

Remember, use the exact same number of words for each summary.
Answer in JSON. The JSON should be a list (length 5) of dictionaries whose keys are "Missing_Entities" and "Denser_Summary".
Use only KOREAN language to reply."""
)


# Create the chain, including
chain = (
    prompt
    | ChatOpenAI(
        temperature=0,
        model="gpt-4-turbo-preview",
        streaming=True,
        callbacks=[StreamCallback()],
    )
    | JsonOutputParser()
    | (lambda x: x[-1]["Denser_Summary"])
)

# Invoke the chain
result = chain.invoke({"ARTICLE": content})
print(result)