better version

app.py

@@ -3,43 +3,108 @@ from concurrent.futures import ThreadPoolExecutor
 import streamlit as st
 import io
 from anthropic import Anthropic
+import tiktoken
+import re
 client = Anthropic()
+encoding_openAI = tiktoken.get_encoding("cl100k_base")
+encoding_anthropic = client.get_tokenizer()
 
-st.sidebar.title("API Configuration")
-api_key = st.sidebar.text_input("Enter your Open API key:")
+# Model choice and max tokens input
+model_choice = st.sidebar.selectbox("Choose a Model", ["OpenAI", "Anthropic"])
+max_tokens = st.sidebar.number_input(
+    "Max number of tokens per chunk", min_value=100, value=8000)
+
+
+def clean_text_content(text):
+    # Keep only English letters, numbers, spaces, line breaks, and common punctuation/symbols
+    cleaned_text = re.sub(r'[^a-zA-Z0-9 \r\n.,;!?()\-\'\"&+:%$#@*]', '', text)
+    return cleaned_text
+
+
+def create_chunks(text, n, tokenizer_name):
+    """Returns successive n-sized chunks from provided text."""
+    tokenizer = encoding_openAI if tokenizer_name == "OpenAI" else encoding_anthropic
+    encoded = tokenizer.encode(text)
+
+    # Check for type of token and adapt accordingly
+    tokens = encoded.ids if hasattr(encoded, "ids") else encoded
+
+    i = 0
+    while i < len(tokens):
+        # Find the nearest end of sentence within a range of 0.5 * n and 1.5 * n tokens
+        j = min(i + int(1.5 * n), len(tokens))
+        while j > i + int(0.5 * n):
+            # Decode the tokens and check for full stop or newline
+            chunk = tokenizer.decode(tokens[i:j])
+            if chunk.endswith(".") or chunk.endswith("\n"):
+                break
+            j -= 1
+        # If no end of sentence found, use n tokens as the chunk size
+        if j == i + int(0.5 * n):
+            j = min(i + n, len(tokens))
+        yield tokens[i:j]
+        i = j
 
 
 def convert_pdf_to_text(pdf_file_data, file_name):
     text = "\n---\n"
     text += f"file name: {file_name}\n content: \n"
     pdf_reader = PdfReader(pdf_file_data)
-    # Extract all text at once
     text += "".join([page.extract_text() for page in pdf_reader.pages])
     text += "\n---\n"
     return text
 
 
 def pdf_to_text(pdf_files_data, file_names):
-    # Create a ThreadPoolExecutor to run the conversion in parallel
     with ThreadPoolExecutor() as executor:
-        # Use the executor to map the convert_pdf_to_text function over all the pdf_files_data
         results = executor.map(convert_pdf_to_text, pdf_files_data, file_names)
-
     return results
 
 
-st.title("PDF to Text Converter")
-st.markdown("Upload PDF files and get their content in text format.")
+st.title("PDF splitter")
+st.markdown(
+    "Upload PDF files and get their content in text format splitted based on the max tokens.")
+
 
-uploaded_files = st.file_uploader(
+uploaded_files = st.sidebar.file_uploader(
     "Upload PDF files", type="pdf", accept_multiple_files=True)
 
+clean_text = st.sidebar.checkbox("Clean text before encoding and splitting?")
+
+# Check if the text is not already in session_state
+if "text_content" not in st.session_state:
+    st.session_state.text_content = ""
+
 if uploaded_files:
     pdf_files_data = [io.BytesIO(uploaded_file.read())
                       for uploaded_file in uploaded_files]
     file_names = [uploaded_file.name for uploaded_file in uploaded_files]
-    if st.button('Convert'):
-        with st.spinner('Converting PDFs...'):
-            text = "\n".join(pdf_to_text(pdf_files_data, file_names))
-        st.text_area("Text content:", text, height=200)
-        st.write(f"Number of tokens: {client.count_tokens(text)}")
+
+    if st.sidebar.button('Convert'):
+        converting_message = st.sidebar.text("Converting PDFs...")
+        converted_text = "\n".join(pdf_to_text(pdf_files_data, file_names))
+        st.session_state.text_content += converted_text
+        converting_message.empty()
+
+    if clean_text:
+        st.session_state.text_content = clean_text_content(
+            st.session_state.text_content)
+
+    chunks_generator = create_chunks(
+        st.session_state.text_content, max_tokens, model_choice)
+    chunks = [encoding_openAI.decode(chunk_tokens) if model_choice == "OpenAI" else encoding_anthropic.decode(
+        chunk_tokens) for chunk_tokens in chunks_generator]
+
+    # Display each chunk in a separate text area
+    for i, chunk in enumerate(chunks, 1):
+        chunk_content = st.text_area(f"Chunk {i} content:", chunk, height=200)
+
+# Button to compute tokens of the text area content
+if st.button("Compute Tokens"):
+    if model_choice == "OpenAI":
+        num_tokens = len(encoding_openAI.encode(st.session_state.text_content))
+        st.write(f"Total number of tokens (OpenAI): {num_tokens}")
+    else:
+        tokens_count = len(encoding_anthropic.encode(
+            st.session_state.text_content))
+        st.write(f"Total number of tokens (Anthropic): {tokens_count}")