How to Scrape Reddit Posts in 2026 (Without Getting Blocked)

// ⚠️ NOT RECOMMENDED: Fails in 2026 due to aggressive IP blocks
async function scrapeSubredditRaw() {
  const url = 'https://www.reddit.com/r/python.json?limit=25';

  const response = await fetch(url, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
    },
  });

  if (!response.ok) {
    throw new Error(`Reddit request rejected: ${response.status} ${response.statusText}`);
  }

  const data = await response.json();
  return data;
}

scrapeSubredditRaw().catch(console.error);

HTTP/1.1 429 Too Many Requests
server: cloudflare
retry-after: 86400
x-ratelimit-remaining: 0
x-ratelimit-reset: 86400
content-type: application/json; charset=UTF-8

{
  "message": "Too Many Requests",
  "error": 429
}

Error: Reddit request rejected: 429 Too Many Requests
    at scrapeSubredditRaw (raw-fetch.ts:12:11)

import { SubScraperClient } from '@subscraper/sdk';

// 1. Initialize client with your API key
const client = new SubScraperClient({
  apiKey: process.env.SUBSCRAPER_API_KEY || 'sk_live_your_api_key',
});

async function main() {
  try {
    // 2. Fetch top posts from r/python over the past month
    const response = await client.getCommunityPosts({
      subreddit: 'python',
      sort: 'top',
      time: 'month',
      limit: 25,
    });

    const posts = response.items ?? [];
    console.log(`Extracted ${posts.length} posts from r/${response.subreddit}\n`);

    // 3. Iterate results and print submission metadata
    for (const post of posts) {
      console.log(`[▲ ${post.score}] ${post.title}`);
      console.log(`  Author:   u/${post.author}`);
      console.log(`  Comments: ${post.commentCount}`);
      console.log(`  URL:      ${post.url}`);
      console.log('--------------------------------------------------');
    }
  } catch (error) {
    console.error('Scraping failed:', error);
  }
}

main();

import { SubScraperClient, PostItem } from '@subscraper/sdk';

const client = new SubScraperClient({
  apiKey: process.env.SUBSCRAPER_API_KEY || 'sk_live_your_api_key',
});

/**
 * Scrapes multiple pages of Reddit posts using cursor anchors
 */
async function scrapeSubredditWithPagination({
  subreddit,
  totalDesired = 100,
}: { subreddit: string; totalDesired?: number }): Promise<PostItem[]> {
  const allPosts: PostItem[] = [];
  let pageCount = 1;

  console.log(`Starting pagination: collecting ${totalDesired} posts from r/${subreddit}...`);

  while (allPosts.length < totalDesired) {
    const remaining = totalDesired - allPosts.length;
    const batchLimit = Math.min(remaining, 25);

    // Execute paginated call via SubScraper API
    const response = await client.getCommunityPosts({
      subreddit,
      sort: 'hot',
      limit: batchLimit,
    });

    const batch = response.items ?? [];
    if (batch.length === 0) {
      console.log('End of feed reached: no further posts available.');
      break;
    }

    allPosts.push(...batch);
    console.log(`Batch ${pageCount}: Added ${batch.length} posts. Progress: ${allPosts.length}/${totalDesired}`);

    // Stop if the response has no next page indicator
    if (!response.nextPageUrl) {
      console.log('No nextPageUrl found. Reached end of listing.');
      break;
    }

    pageCount++;

    // Polite 300ms delay to prevent client-side queue congestion
    await new Promise((resolve) => setTimeout(resolve, 300));
  }

  return allPosts;
}

scrapeSubredditWithPagination({ subreddit: 'MachineLearning', totalDesired: 50 })
  .then((posts) => console.log(`Done! Collected ${posts.length} posts.`))
  .catch(console.error);