Introduction to Web Scraping and FireCrawl Setup

FIRECRAWL_API_KEY=your_api_key_here
NODE_ENV=development

node_modules/
.env
.env.local
.env.production
dist/
build/
*.log
data/output/*.json
data/output/*.csv
.DS_Store

import FirecrawlApp from '@mendable/firecrawl-js';
import dotenv from 'dotenv';

dotenv.config();

if (!process.env.FIRECRAWL_API_KEY) {
  throw new Error('FIRECRAWL_API_KEY environment variable is required');
}

export const firecrawlApp = new FirecrawlApp({
  apiKey: process.env.FIRECRAWL_API_KEY,
});

export const defaultScrapeOptions = {
  formats: ['markdown', 'html'] as const,
  includeTags: ['title', 'meta', 'h1', 'h2', 'h3', 'p', 'a', 'img'],
  excludeTags: ['script', 'style', 'nav', 'footer'],
  onlyMainContent: true,
  timeout: 30000,
};

export const defaultCrawlOptions = {
  limit: 10,
  scrapeOptions: defaultScrapeOptions,
  allowBackwardCrawling: false,
  allowExternalContentLinks: false,
};

import { firecrawlApp, defaultScrapeOptions } from '../config/firecrawl';
import { saveToFile } from '../utils/helpers';

export interface ScrapedData {
  url: string;
  title: string;
  content: string;
  metadata: {
    scrapedAt: string;
    statusCode: number;
    contentLength: number;
  };
}

export class BasicScraper {
  async scrapeUrl(url: string): Promise<ScrapedData | null> {
    try {
      console.log(`🔍 Scraping: ${url}`);

      const scrapeResult = await firecrawlApp.scrapeUrl(url, {
        ...defaultScrapeOptions,
        formats: ['markdown'],
      });

      if (!scrapeResult.success) {
        console.error('❌ Scraping failed:', scrapeResult.error);
        return null;
      }

      const data: ScrapedData = {
        url,
        title: scrapeResult.data.metadata?.title || 'No title',
        content: scrapeResult.data.markdown || '',
        metadata: {
          scrapedAt: new Date().toISOString(),
          statusCode: scrapeResult.data.metadata?.statusCode || 0,
          contentLength: scrapeResult.data.markdown?.length || 0,
        },
      };

      console.log(`✅ Successfully scraped: ${data.title}`);
      return data;

    } catch (error) {
      console.error('❌ Error scraping URL:', error);
      return null;
    }
  }

  async scrapeMultipleUrls(urls: string[]): Promise<ScrapedData[]> {
    const results: ScrapedData[] = [];

    for (const url of urls) {
      const data = await this.scrapeUrl(url);
      if (data) {
        results.push(data);
        // Add delay to respect rate limits
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
    }

    return results;
  }
}

import fs from 'fs/promises';
import path from 'path';

export async function saveToFile(
  data: any,
  filename: string,
  format: 'json' | 'csv' = 'json'
): Promise<void> {
  const outputDir = path.join(process.cwd(), 'data', 'output');

  // Ensure output directory exists
  await fs.mkdir(outputDir, { recursive: true });

  const filePath = path.join(outputDir, `${filename}.${format}`);

  try {
    if (format === 'json') {
      await fs.writeFile(filePath, JSON.stringify(data, null, 2));
    } else if (format === 'csv') {
      // Simple CSV conversion (you might want to use a library like csv-writer)
      const csv = convertToCSV(data);
      await fs.writeFile(filePath, csv);
    }

    console.log(`💾 Data saved to: ${filePath}`);
  } catch (error) {
    console.error('❌ Error saving file:', error);
  }
}

function convertToCSV(data: any[]): string {
  if (!data.length) return '';

  const headers = Object.keys(data[0]);
  const csvHeaders = headers.join(',');

  const csvRows = data.map(row =>
    headers.map(header => {
      const value = row[header];
      // Handle nested objects and escape commas
      if (typeof value === 'object') {
        return `"${JSON.stringify(value).replace(/"/g, '""')}"`;
      }
      return `"${String(value).replace(/"/g, '""')}"`;
    }).join(',')
  );

  return [csvHeaders, ...csvRows].join('\n');
}

export function delay(ms: number): Promise<void> {
  return new Promise(resolve => setTimeout(resolve, ms));
}

export function isValidUrl(url: string): boolean {
  try {
    new URL(url);
    return true;
  } catch {
    return false;
  }
}

import { BasicScraper } from './scrapers/basic-scraper';
import { saveToFile } from './utils/helpers';

async function main() {
  const scraper = new BasicScraper();

  // Test URLs - replace with your targets
  const testUrls = [
    'https://example.com',
    'https://httpbin.org/html',
    'https://quotes.toscrape.com/',
  ];

  console.log('🚀 Starting web scraping test...');

  const results = await scraper.scrapeMultipleUrls(testUrls);

  if (results.length > 0) {
    await saveToFile(results, `scraping-test-${Date.now()}`, 'json');
    console.log(`✅ Scraped ${results.length} pages successfully!`);
  } else {
    console.log('❌ No data was scraped');
  }
}

// Run the script
main().catch(console.error);