Trove Research Harvester
Results Summary
Results found: 0 articles
Show article list
Clear list
Harvest OCR Text
Each article is processed with a short delay to avoid triggering Trove’s robot‑blocking system.
Start Harvesting
Save Results So Far
Status: Not started.
This tool runs entirely in your browser.
Created independently by Dave for heritage researchers.
Not affiliated with CRHR or any organisation.
', jsonStart);
if (jsonEnd === -1) return null;
const jsonText = html.slice(jsonStart, jsonEnd).trim();
try {
return JSON.parse(jsonText);
} catch (e) {
console.error('JSON parse error:', e);
return null;
}
}
// Extract article data from the JSON
function extractArticleData(json) {
try {
const article = json.props.pageProps.article;
return {
id: article.id || '',
title: article.title || '',
date: article.date || '',
newspaper: article.newspaper || '',
troveOCR: article.articleText || '',
imageUrl: article.imageUrl || '',
zone: article.zone || null
};
} catch (e) {
console.error('Article extraction error:', e);
return null;
}
}
// Harvest controller
async function startHarvest() {
if (articles.length === 0) {
alert('No articles to harvest. Run a search first.');
return;
}
harvesting = true;
currentIndex = 0;
document.getElementById('status-text').textContent =
`Status: Reading article 1 of ${articles.length}…`;
updateProgressBar();
document.getElementById('streaming-panel').innerHTML = '';
csvRows = [];
harvestNextArticle();
}
async function harvestNextArticle() {
if (!harvesting || currentIndex >= articles.length) {
harvesting = false;
document.getElementById('status-text').textContent = 'Status: Finished.';
updateProgressBar();
return;
}
const article = articles[currentIndex];
try {
const html = await fetchTroveHTML(article.url);
const json = extractEmbeddedJSON(html);
const data = json ? extractArticleData(json) : null;
if (data) {
article.id = data.id || article.id;
article.title = data.title || article.title;
article.date = data.date || article.date;
article.newspaper = data.newspaper || article.newspaper;
article.troveOCR = data.troveOCR || '';
article.imageUrl = data.imageUrl || '';
article.zone = data.zone || null;
} else {
article.troveOCR = '(No OCR found in embedded JSON.)';
}
appendToStreamingPanel(article);
addArticleToCSV(article);
} catch (e) {
console.error(e);
article.troveOCR = '(Error fetching article HTML or parsing JSON.)';
appendToStreamingPanel(article);
addArticleToCSV(article);
}
currentIndex++;
updateProgressBar();
if (currentIndex < articles.length) {
document.getElementById('status-text').textContent =
`Status: Reading article ${currentIndex + 1} of ${articles.length}…`;
setTimeout(harvestNextArticle, 6000); // fixed 6 seconds
} else {
document.getElementById('status-text').textContent = 'Status: Finished.';
}
}
function updateProgressBar() {
const bar = document.getElementById('progress-bar');
if (articles.length === 0) {
bar.style.width = '0%';
return;
}
const pct = Math.min(100, (currentIndex / articles.length) * 100);
bar.style.width = pct + '%';
}
// Streaming display
function appendToStreamingPanel(article) {
const panel = document.getElementById('streaming-panel');
const block = document.createElement('div');
block.className = 'article-block';
block.id = 'article-' + article.id;
const header = document.createElement('div');
header.className = 'article-header';
const headerText = `${article.title || '(Untitled)'} — ${article.newspaper || ''} — ${article.date || ''}`;
header.textContent = headerText;
block.appendChild(header);
const preview = document.createElement('div');
preview.className = 'ocr-preview';
preview.textContent = truncateTo500Chars(article.troveOCR);
block.appendChild(preview);
const btnRow = document.createElement('div');
const showMoreBtn = document.createElement('button');
showMoreBtn.className = 'button-secondary';
showMoreBtn.textContent = 'Show more';
showMoreBtn.onclick = () => {
preview.textContent = article.troveOCR;
};
btnRow.appendChild(showMoreBtn);
const openTroveBtn = document.createElement('button');
openTroveBtn.className = 'button-secondary';
openTroveBtn.textContent = 'Open in Trove';
openTroveBtn.onclick = () => {
window.open(article.url, '_blank');
};
btnRow.appendChild(openTroveBtn);
const modernBtn = document.createElement('button');
modernBtn.className = 'button-primary';
modernBtn.textContent = 'Fetch modern OCR';
modernBtn.onclick = () => fetchModernOCR(article);
btnRow.appendChild(modernBtn);
block.appendChild(btnRow);
const modernDiv = document.createElement('div');
modernDiv.className = 'modern-ocr';
modernDiv.id = 'modern-ocr-' + article.id;
modernDiv.textContent = 'Modern OCR: (Not fetched yet)';
block.appendChild(modernDiv);
panel.appendChild(block);
}
function truncateTo500Chars(text) {
if (!text) return '';
if (text.length <= 500) return text;
return text.slice(0, 500) + '…';
}
// Modern OCR using full page image (fallback)
async function fetchModernOCR(article) {
const modernDiv = document.getElementById('modern-ocr-' + article.id);
modernDiv.textContent = 'Modern OCR: Fetching…';
try {
const fullPageImageURL = article.imageUrl ||
'https://via.placeholder.com/800x1200?text=No+image+URL+found';
const result = await Tesseract.recognize(
fullPageImageURL,
'eng',
{ logger: m => {} }
);
const text = result.data.text || '';
article.modernOCR = text;
modernDiv.textContent = 'Modern OCR:\n' + text;
addArticleToCSV(article);
} catch (e) {
modernDiv.textContent = 'Modern OCR: Error fetching or processing image.';
console.error(e);
}
}
// CSV builder
function addArticleToCSV(article) {
const existingIndex = csvRows.findIndex(r => r.id === article.id);
const rowData = {
id: article.id,
title: article.title,
date: article.date,
newspaper: article.newspaper,
url: article.url,
troveOCR: article.troveOCR || '',
modernOCR: article.modernOCR || ''
};
if (existingIndex >= 0) {
csvRows[existingIndex] = rowData;
} else {
csvRows.push(rowData);
}
}
function downloadCSV(allRows) {
if (!allRows || allRows.length === 0) {
alert('No data to save.');
return;
}
const header = ['id','title','date','newspaper','url','troveOCR','modernOCR'];
const lines = [header.join(',')];
allRows.forEach(r => {
const line = [
r.id,
escapeCSV(r.title),
r.date,
escapeCSV(r.newspaper),
r.url,
escapeCSV(r.troveOCR),
escapeCSV(r.modernOCR)
].join(',');
lines.push(line);
});
const blob = new Blob([lines.join('\n')], { type: 'text/csv;charset=utf-8;' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'trove_research_harvester.csv';
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
}
function escapeCSV(text) {
if (text == null) return '';
const t = String(text).replace(/"/g, '""');
return `"${t}"`;
}
function downloadPartialCSV() {
downloadCSV(csvRows);
}
// Event wiring
document.getElementById('run-search-btn').addEventListener('click', fetchSearchResults);
document.getElementById('start-harvest-btn').addEventListener('click', startHarvest);
document.getElementById('save-partial-btn').addEventListener('click', downloadPartialCSV);
document.getElementById('show-list-btn').addEventListener('click', () => {
const listEl = document.getElementById('article-list');
listEl.style.display = listEl.style.display === 'none' ? 'block' : 'none';
});
document.getElementById('clear-list-btn').addEventListener('click', () => {
articles = [];
csvRows = [];
document.getElementById('article-list').innerHTML = '';
updateResultsSummary();
document.getElementById('streaming-panel').innerHTML = '';
document.getElementById('status-text').textContent = 'Status: Not started.';
updateProgressBar();
});