Collect Reddit Training Data for LLMs at Scale

import { SubScraperClient } from '@subscraper/sdk';
import * as fs from 'fs';

const client = new SubScraperClient({
  apiKey: process.env.SUBSCRAPER_API_KEY!
});

// Collect top-voted Q&A pairs from r/AskScience with pagination
async function collectAskScienceQAPairs(maxPages = 5) {
  const dataset: Array<{ question: string; answer: string; score: number }> = [];
  let afterToken: string | undefined = undefined;

  for (let page = 0; page < maxPages; page++) {
    // 1. Fetch top posts for the year
    const feed = await client.getCommunityPosts({
      subreddit: 'askscience',
      sort: 'top',
      time: 'year',
      limit: 50,
      ...(afterToken ? { after: afterToken } : {})
    });

    for (const post of feed.posts) {
      if (post.score < 50 || !post.url) continue;

      // 2. Fetch full comment tree for top answer
      const thread = await client.getPost({
        url: post.url,
        commentLimit: 25
      });

      // 3. Filter high-scoring, non-deleted comments
      const bestAnswer = thread.comments
        .filter(c => !c.isDeleted && c.score >= 25 && c.body.length > 100)
        .sort((a, b) => b.score - a.score)[0];

      if (bestAnswer) {
        dataset.push({
          question: &grave;${post.title}\n\n${post.body || ''}&grave;.trim(),
          answer: bestAnswer.body,
          score: bestAnswer.score
        });
      }
    }

    afterToken = feed.after;
    if (!afterToken) break;
  }

  // 4. Save to JSONL for LLM fine-tuning
  const lines = dataset.map(d => JSON.stringify({
    messages: [
      { role: 'user', content: d.question },
      { role: 'assistant', content: d.answer }
    ]
  })).join('\n');

  fs.writeFileSync('askscience_qa.jsonl', lines);
  console.log(&grave;Saved ${dataset.length} QA pairs!&grave;);
}