Firecrawl実践ガイド - WebサイトをLLM対応Markdownに変換する最強ツール

# pipを使用
pip install firecrawl-py

# poetryを使用
poetry add firecrawl-py

# 環境変数の設定
export FIRECRAWL_API_KEY="your-api-key-here"

# Pythonでの初期化
from firecrawl import FirecrawlApp

app = FirecrawlApp(api_key=os.environ.get('FIRECRAWL_API_KEY'))

# Go modulesを使用
go get github.com/mendableai/firecrawl-go

# 環境変数の設定
export FIRECRAWL_API_KEY="your-api-key-here"

// Goでの初期化
package main

import (
    "github.com/mendableai/firecrawl-go"
    "os"
)

func main() {
    apiKey := os.Getenv("FIRECRAWL_API_KEY")
    app := firecrawl.New(apiKey)
}

# 基本的なcURL呼び出し
curl -X POST https://api.firecrawl.dev/v0/scrape \
  -H "Authorization: Bearer $FIRECRAWL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "pageOptions": {
      "onlyMainContent": true
    }
  }'

// データクリーニングとフィルタリング
interface CleaningRule {
  name: string;
  apply: (content: string) => string;
  validate: (content: string) => boolean;
}

class DataCleaner {
  private rules: CleaningRule[] = [
    {
      name: 'remove_email_addresses',
      apply: (content) => content.replace(/[\w.-]+@[\w.-]+\.\w+/g, '[EMAIL]'),
      validate: (content) => !content.match(/[\w.-]+@[\w.-]+\.\w+/),
    },
    {
      name: 'remove_phone_numbers',
      apply: (content) => content.replace(/\+?\d{1,4}?[-.\s]?\(?\d{1,3}?\)?[-.\s]?\d{1,4}[-.\s]?\d{1,4}[-.\s]?\d{1,9}/g, '[PHONE]'),
      validate: (content) => true,
    },
    {
      name: 'normalize_whitespace',
      apply: (content) => content.replace(/\s+/g, ' ').trim(),
      validate: (content) => !content.match(/\s{2,}/),
    },
    {
      name: 'remove_special_chars',
      apply: (content) => content.replace(/[^\w\s\-.,!?'"]/g, ''),
      validate: (content) => true,
    },
  ];
  
  async cleanDataset(data: any[]) {
    const cleaned = [];
    
    for (const item of data) {
      let content = item.content;
      
      // 各ルールを適用
      for (const rule of this.rules) {
        content = rule.apply(content);
      }
      
      // 最小文字数チェック
      if (content.length < 100) {
        console.log(`Skipping short content: ${item.url}`);
        continue;
      }
      
      // 言語検出（オプション）
      const language = await this.detectLanguage(content);
      if (language !== 'en' && language !== 'ja') {
        console.log(`Skipping non-target language: ${language}`);
        continue;
      }
      
      cleaned.push({
        ...item,
        content,
        metadata: {
          ...item.metadata,
          language,
          cleanedAt: new Date().toISOString(),
        },
      });
    }
    
    return cleaned;
  }
  
  private async detectLanguage(text: string): Promise<string> {
    // 簡易的な言語検出（実際はライブラリを使用）
    const japanesePattern = /[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FAF]/;
    return japanesePattern.test(text) ? 'ja' : 'en';
  }
}

// 構造化とフォーマット変換
interface StructuredData {
  id: string;
  source: string;
  content: string;
  metadata: Record<string, any>;
  embeddings?: number[];
}

class DataStructurer {
  async structureForLLM(data: any[]): Promise<StructuredData[]> {
    const structured = [];
    
    for (const item of data) {
      // チャンク分割
      const chunks = this.splitIntoChunks(item.content, {
        maxTokens: 2048,
        overlap: 200,
      });
      
      for (const [index, chunk] of chunks.entries()) {
        structured.push({
          id: `${item.url}-chunk-${index}`,
          source: item.url,
          content: chunk,
          metadata: {
            ...item.metadata,
            chunkIndex: index,
            totalChunks: chunks.length,
            domain: new URL(item.url).hostname,
          },
        });
      }
    }
    
    return structured;
  }
  
  private splitIntoChunks(
    text: string,
    options: { maxTokens: number; overlap: number }
  ): string[] {
    const chunks: string[] = [];
    const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
    
    let currentChunk = '';
    let currentTokens = 0;
    
    for (const sentence of sentences) {
      const sentenceTokens = this.estimateTokens(sentence);
      
      if (currentTokens + sentenceTokens > options.maxTokens) {
        if (currentChunk) {
          chunks.push(currentChunk.trim());
          
          // オーバーラップ処理
          const overlapSentences = currentChunk
            .split(/[.!?]+/)
            .slice(-2)
            .join('. ');
          currentChunk = overlapSentences + ' ' + sentence;
          currentTokens = this.estimateTokens(currentChunk);
        }
      } else {
        currentChunk += ' ' + sentence;
        currentTokens += sentenceTokens;
      }
    }
    
    if (currentChunk.trim()) {
      chunks.push(currentChunk.trim());
    }
    
    return chunks;
  }
  
  private estimateTokens(text: string): number {
    // 簡易的なトークン推定（実際はtiktokenなどを使用）
    return Math.ceil(text.split(/\s+/).length * 1.3);
  }
  
  async exportToJSONL(data: StructuredData[], filepath: string) {
    const jsonl = data
      .map(item => JSON.stringify({
        text: item.content,
        metadata: item.metadata,
      }))
      .join('\n');
    
    await writeFile(filepath, jsonl, 'utf-8');
  }
}

// データ品質バリデーション
interface ValidationResult {
  valid: boolean;
  errors: string[];
  warnings: string[];
  stats: {
    totalItems: number;
    validItems: number;
    avgContentLength: number;
    uniqueDomains: number;
  };
}

class DataValidator {
  async validateDataset(data: StructuredData[]): Promise<ValidationResult> {
    const errors: string[] = [];
    const warnings: string[] = [];
    const validItems: string[] = [];
    
    // 基本的な統計情報
    const stats = {
      totalItems: data.length,
      validItems: 0,
      avgContentLength: 0,
      uniqueDomains: new Set(data.map(d => d.metadata.domain)).size,
    };
    
    // 各アイテムのバリデーション
    for (const [index, item of data.entries()) {
      const itemErrors: string[] = [];
      
      // 必須フィールドチェック
      if (!item.id) {
        itemErrors.push(`Item ${index}: Missing ID`);
      }
      if (!item.content || item.content.trim().length === 0) {
        itemErrors.push(`Item ${index}: Empty content`);
      }
      
      // コンテンツ品質チェック
      if (item.content.length < 50) {
        warnings.push(`Item ${index}: Very short content (${item.content.length} chars)`);
      }
      
      // 重複チェック
      const contentHash = this.hashContent(item.content);
      if (this.seenHashes.has(contentHash)) {
        warnings.push(`Item ${index}: Duplicate content detected`);
      }
      this.seenHashes.add(contentHash);
      
      // 文字化けチェック
      if (this.hasGarbledText(item.content)) {
        itemErrors.push(`Item ${index}: Contains garbled text`);
      }
      
      if (itemErrors.length === 0) {
        stats.validItems++;
        validItems.push(item.id);
      } else {
        errors.push(...itemErrors);
      }
    }
    
    // 平均コンテンツ長の計算
    stats.avgContentLength = Math.round(
      data.reduce((sum, item) => sum + item.content.length, 0) / data.length
    );
    
    return {
      valid: errors.length === 0,
      errors,
      warnings,
      stats,
    };
  }
  
  private seenHashes = new Set<string>();
  
  private hashContent(content: string): string {
    // 簡易的なハッシュ（実際はcrypto.createHashを使用）
    return content.slice(0, 100).toLowerCase().replace(/\s+/g, '');
  }
  
  private hasGarbledText(text: string): boolean {
    // 文字化けの検出
    const garbledPatterns = [
      /[\x00-\x1F\x7F-\x9F]/,  // 制御文字
      /�{3,}/,  // 連続する置換文字
      /[^\x00-\x7F\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FAF\uFF00-\uFFEF]/,  // 想定外の文字
    ];
    
    return garbledPatterns.some(pattern => pattern.test(text));
  }
}