팁 & FAQ

"""
Requirements: `pip install pymupdf` to import fitz
"""

import os
import fitz

def split_pdf(input_file, batch_size):
    # input_pdf 열기
    input_pdf = fitz.open(input_file)
    num_pages = len(input_pdf)
    print(f"Total number of pages: {num_pages}")

    # input_pdf 분할
    for start_page in range(0, num_pages, batch_size):
        end_page = min(start_page + batch_size, num_pages) - 1

        # output_pdf를 파일로 저장
        input_file_basename = os.path.splitext(input_file)[0]
        output_file = f"{input_file_basename}_{start_page}_{end_page}.pdf"
        print(output_file)
        with fitz.open() as output_pdf:
            output_pdf.insert_pdf(input_pdf, from_page=start_page, to_page=end_page)
            output_pdf.save(output_file)

    # input_pdf 닫기
    input_pdf.close()

# 입력 파라미터
input_file = "paper.pdf"  # 본인 파일로 교체하세요
batch_size = 10  # 최대 사용 가능 값은 100
split_pdf(input_file, batch_size)

"""
Requirements: `pip install requests`
"""

from glob import glob
import json
import os
import requests

API_KEY = "UPSTAGE_API_KEY"

def call_document_parse(input_file, output_file):
    # 요청 전송
    response = requests.post(
        "https://api.upstage.ai/v1/document-digitization",
        headers={"Authorization": f"Bearer {API_KEY}"},
        data={"base64_encoding": "['figure']", "model": "document-parse"}, # figure 카테고리의 잘라낸 이미지를 base64로 인코딩
        files={"document": open(input_file, "rb")})

    # 응답 저장
    if response.status_code == 200:
        with open(output_file, "w") as f:
            json.dump(response.json(), f, ensure_ascii=False)
    else:
        raise ValueError(f"Unexpected status code {response.status_code}.")

# input_file과 관련된 모든 짧은 PDF 찾기
input_file = "paper.pdf"
short_input_files = glob(os.path.splitext(input_file)[0] + "_*.pdf")

# 모든 짧은 PDF에 대해 요청을 전송하고 응답 저장
for short_input_file in short_input_files:
    print(short_input_file)
    short_output_file = os.path.splitext(short_input_file)[0] + ".json"
    call_document_parse(short_input_file, short_output_file)

import json
import base64

# 입력 파라미터
json_file = "paper_0_9.json"
output_file = "paper_cropped_figure.png"

# JSON 파일을 로드해 첫 번째 figure의 바운딩 박스를 가져옴
with open(json_file, "r") as f:
    data = json.load(f)

    # 첫 번째 figure의 바운딩 박스를 가져와 이미지를 잘라냄
    for element in data["elements"]:
        if element["category"] == "figure":
            with open (output_file, 'wb') as fh:
                fh.write(base64.decodebytes(str.encode(element["base64_encoding"])))
            break

def get_chunks(lines, max_context_length=4096):
    chunks = []
    current_chunk = []

    for line in lines:
        if len(" ".join(current_chunk + [line])) <= max_context_length:
            current_chunk.append(line)
        else:
            chunks.append(current_chunk)
            current_chunk = [line]

    if current_chunk:
        chunks.append(current_chunk)
    return chunks

with open("paper.html", "r") as f:
    lines = f.read().split("\n")

chunks = get_chunks(lines, max_context_length=1024)
print(chunks[0])