#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
VIDEHA & SADEHA PDF SEARCH SERVER - FIXED VERSION
Handles special characters and Unicode errors
"""
import os
import sys
import json
import re
import glob
import traceback
from pathlib import Path
from http.server import HTTPServer, BaseHTTPRequestHandler
import urllib.parse
# Try to use pypdf for text extraction
try:
from pypdf import PdfReader
HAS_PYPDF = True
print("✅ pypdf loaded successfully")
except ImportError:
HAS_PYPDF = False
print("⚠️ pypdf not installed. Run: pip install pypdf")
PORT = 8080
INDEX_FILE = "pdf_index.json"
def safe_json_dump(data, filepath):
"""Safely save JSON data, handling Unicode errors"""
try:
with open(filepath, 'w', encoding='utf-8', errors='ignore') as f:
json.dump(data, f, ensure_ascii=False, indent=2, default=str)
return True
except Exception as e:
print(f"⚠️ JSON save error: {e}")
# Try with ascii encoding only
try:
with open(filepath, 'w', encoding='ascii', errors='ignore') as f:
json.dump(data, f, ensure_ascii=True, indent=2, default=str)
print(" Saved with ASCII encoding")
return True
except Exception as e2:
print(f"❌ Failed to save index: {e2}")
return False
def clean_text(text):
"""Clean text by removing invalid Unicode characters"""
if not text:
return ""
# Replace invalid surrogate pairs
text = text.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore')
# Remove control characters except newlines and tabs
text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text)
return text
class PDFSearchEngine:
def __init__(self, folder_path):
self.folder_path = Path(folder_path)
self.index = {}
def extract_text_from_pdf(self, pdf_path):
"""Extract text from PDF using pypdf with error handling"""
if not HAS_PYPDF:
return ""
try:
reader = PdfReader(str(pdf_path))
text = ""
num_pages = len(reader.pages)
for page_num, page in enumerate(reader.pages):
try:
page_text = page.extract_text()
if page_text:
text += clean_text(page_text) + " "
except Exception as e:
# Skip problematic pages
pass
# Show progress
if (page_num + 1) % 10 == 0 or page_num + 1 == num_pages:
print(f" Page {page_num+1}/{num_pages}", end="\r")
return clean_text(text)
except Exception as e:
print(f"\n ⚠️ Error reading PDF: {e}")
return ""
def build_index(self):
print("\n" + "="*60)
print("📚 Building PDF index with pypdf...")
print("="*60)
print(f"📂 Folder: {self.folder_path}")
if not self.folder_path.exists():
print(f"❌ Folder not found!")
return False
pdf_files = list(self.folder_path.glob("*.pdf")) + list(self.folder_path.glob("*.PDF"))
pdf_files = sorted(pdf_files)
print(f"📄 Found {len(pdf_files)} PDF files")
if len(pdf_files) == 0:
print("❌ No PDF files found!")
return False
print("\n📖 Extracting text from PDFs (this will take 5-10 minutes for 478 files)...")
print(" Each dot = 1 file processed\n")
for i, pdf_path in enumerate(pdf_files):
file_size = pdf_path.stat().st_size / (1024 * 1024)
filename = pdf_path.name.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore')
print(f" [{i+1}/{len(pdf_files)}] {filename[:60]} ({file_size:.1f} MB)")
text = ""
if HAS_PYPDF:
text = self.extract_text_from_pdf(pdf_path)
if text:
# Limit text size to avoid memory issues
text = text[:150000]
print(f" ✅ {len(text)} characters extracted")
else:
print(f" ⚠️ No text extracted")
# Store in index with clean keys
safe_filename = filename
self.index[safe_filename] = {
"path": str(pdf_path).encode('utf-8', errors='ignore').decode('utf-8', errors='ignore'),
"name": safe_filename,
"text": text[:150000] if text else "",
"size_mb": round(file_size, 2),
"has_text": bool(text)
}
# Save index with error handling
print("\n💾 Saving index...")
if safe_json_dump(self.index, INDEX_FILE):
print(f"✅ Index saved to {INDEX_FILE}")
else:
print("⚠️ Could not save index, but search will work in memory")
text_count = sum(1 for d in self.index.values() if d.get("has_text"))
print(f"\n📊 Total files indexed: {len(self.index)}")
print(f"📖 Files with extractable text: {text_count}/{len(self.index)}")
if not HAS_PYPDF:
print("\n⚠️ To enable full text search, install pypdf:")
print(' C:\\Users\\DELL\\AppData\\Local\\Python\\pythoncore-3.14-64\\Scripts\\pip.exe install pypdf')
print("="*60)
return True
def load_index(self):
if os.path.exists(INDEX_FILE):
try:
with open(INDEX_FILE, 'r', encoding='utf-8', errors='ignore') as f:
self.index = json.load(f)
text_count = sum(1 for d in self.index.values() if d.get("has_text", False))
print(f"✅ Loaded {len(self.index)} PDFs from index")
print(f"📖 Files with text content: {text_count}")
return True
except Exception as e:
print(f"⚠️ Could not load index: {e}")
return False
return False
def search(self, query):
"""Search through indexed PDFs"""
if not query:
return []
query_lower = query.lower()
results = []
for filename, data in self.index.items():
match_found = False
context = ""
count = 0
# Search in text content if available
if data.get("text"):
try:
text_lower = data["text"].lower()
if query_lower in text_lower:
match_found = True
count = text_lower.count(query_lower)
# Find context around first match
pos = text_lower.find(query_lower)
start = max(0, pos - 150)
end = min(len(data["text"]), pos + 200)
context = data["text"][start:end] if pos >= 0 else ""
context = clean_text(context)
except Exception:
pass
# Also search in filename
if not match_found and query_lower in filename.lower():
match_found = True
count = 1
context = f"Filename matches: {filename}"
if match_found:
results.append({
"filename": filename,
"path": data["path"],
"name": data["name"],
"context": context[:300] if context else "",
"count": count,
"size_mb": data.get("size_mb", 0),
"has_text": data.get("has_text", False)
})
results.sort(key=lambda x: x["count"], reverse=True)
return results
class SearchHandler(BaseHTTPRequestHandler):
engine = None
def log_message(self, format, *args):
pass
def do_GET(self):
try:
if self.path == "/" or self.path == "/index.html":
self.serve_html()
elif self.path.startswith("/search"):
self.handle_search()
elif self.path == "/status":
self.send_status()
else:
self.send_response(404)
self.end_headers()
except Exception as e:
print(f"Error: {e}")
self.send_response(500)
self.end_headers()
def send_status(self):
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
text_count = sum(1 for d in self.engine.index.values() if d.get("has_text", False)) if self.engine else 0
status = {
"status": "running",
"pdf_count": len(self.engine.index) if self.engine else 0,
"has_full_text": text_count > 0 if self.engine else False,
"text_files": text_count
}
try:
self.wfile.write(json.dumps(status).encode('utf-8', errors='ignore'))
except:
self.wfile.write(b'{"status":"running"}')
def serve_html(self):
html = '''
विदेह आ सदेह | PDF ताकू इंजिन
📚 विदेह & सदेह
प्रथम मैथिली पाक्षिक ई पत्रिका — पूरा पाठ PDF ताकू इंजिन
⏳ लोड होइत अछि...
🔍 478 PDF अंक के भीतर ताकू
मैथिलीकवितागद्यविदेहसदेह
🔍 ऊपर देल गेल खोज बॉक्समे लिखू आ ताकू कुंजी दबाउ।
'''
self.send_response(200)
self.send_header('Content-type', 'text/html; charset=utf-8')
self.end_headers()
self.wfile.write(html.encode('utf-8', errors='ignore'))
def handle_search(self):
parsed = urllib.parse.urlparse(self.path)
query = urllib.parse.parse_qs(parsed.query).get('q', [''])[0]
results = self.engine.search(query) if self.engine and query else []
self.send_response(200)
self.send_header('Content-type', 'application/json; charset=utf-8')
self.end_headers()
try:
self.wfile.write(json.dumps({"results": results}, ensure_ascii=False, default=str).encode('utf-8', errors='ignore'))
except:
self.wfile.write(b'{"results":[]}')
def main():
pdf_folder = r"C:\Users\DELL\Desktop\Videha PDFs"
print("\n" + "="*60)
print("📚 VIDEHA & SADEHA PDF SEARCH SERVER (Fixed)")
print("="*60)
print(f"📂 PDF Folder: {pdf_folder}")
if not os.path.exists(pdf_folder):
print(f"\n❌ Folder not found: {pdf_folder}")
return
# Delete old corrupted index if exists
if os.path.exists(INDEX_FILE):
try:
os.remove(INDEX_FILE)
print("🗑️ Removed old index file")
except:
pass
engine = PDFSearchEngine(pdf_folder)
# Build new index
print("\n📖 Building search index...")
print("⏳ This will take 5-10 minutes for 478 PDFs...")
engine.build_index()
SearchHandler.engine = engine
print("\n" + "="*60)
print(f"🚀 SERVER RUNNING!")
print(f"📡 Open in browser: http://localhost:{PORT}")
print(f"📁 PDF Folder: {pdf_folder}")
print(f"📊 Indexed PDFs: {len(engine.index)}")
text_count = sum(1 for d in engine.index.values() if d.get("has_text", False))
if text_count > 0:
print(f"📖 Full text search: ENABLED ({text_count} files with text)")
else:
print(f"⚠️ Full text search: DISABLED")
print("="*60)
print("\n🛑 Press Ctrl+C to stop the server\n")
server = HTTPServer(('localhost', PORT), SearchHandler)
try:
server.serve_forever()
except KeyboardInterrupt:
print("\n\n👋 Server stopped.")
if __name__ == "__main__":
main()