5 Ways to Use CrawlForge with LangChain: AI Web Scraping Tutorial

npm install langchain @anthropic-ai/sdk dotenv

# .env
ANTHROPIC_API_KEY=sk-ant-xxxxx
CRAWLFORGE_API_KEY=cf_live_xxxxx

import { ChatAnthropic } from "@langchain/anthropic";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { MemoryVectorStore } from "langchain/vectorstores/memory";
import { OpenAIEmbeddings } from "@langchain/openai";

// CrawlForge client helper
async function fetchWebContent(url: string): Promise<string> {
  const response = await fetch('https://crawlforge.dev/api/v1/tools/extract_content', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ url })
  });

  const data = await response.json();
  return data.content || '';
}

// Web-augmented RAG
async function webAugmentedRAG(query: string, urls: string[]) {
  // 1. Fetch web content (2 credits per URL)
  const contents = await Promise.all(urls.map(fetchWebContent));

  // 2. Split into chunks
  const splitter = new RecursiveCharacterTextSplitter({
    chunkSize: 1000,
    chunkOverlap: 200
  });

  const docs = await splitter.createDocuments(contents);

  // 3. Create vector store
  const vectorStore = await MemoryVectorStore.fromDocuments(
    docs,
    new OpenAIEmbeddings()
  );

  // 4. Retrieve relevant chunks
  const relevantDocs = await vectorStore.similaritySearch(query, 4);

  // 5. Generate answer with context
  const llm = new ChatAnthropic({
    modelName: "claude-3-5-sonnet-20241022"
  });

  const context = relevantDocs.map(d => d.pageContent).join("\n\n");

  const response = await llm.invoke([
    {
      role: "system",
      content: "Answer based on the provided context. Cite sources when possible."
    },
    {
      role: "user",
      content: `Context:\n${context}\n\nQuestion: ${query}`
    }
  ]);

  return response.content;
}

// Usage
const answer = await webAugmentedRAG(
  "What are the new features in Next.js 15?",
  [
    "https://nextjs.org/blog/next-15",
    "https://nextjs.org/docs/app/building-your-application/upgrading"
  ]
);

import { ChatAnthropic } from "@langchain/anthropic";
import { DynamicStructuredTool } from "@langchain/core/tools";
import { AgentExecutor, createOpenAIFunctionsAgent } from "langchain/agents";
import { z } from "zod";

// Define CrawlForge tools for the agent
const searchTool = new DynamicStructuredTool({
  name: "search_web",
  description: "Search the web for information on a topic",
  schema: z.object({
    query: z.string().describe("The search query"),
    limit: z.number().optional().describe("Number of results (default 5)")
  }),
  func: async ({ query, limit = 5 }) => {
    const response = await fetch('https://crawlforge.dev/api/v1/tools/search_web', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ query, limit })
    });

    const data = await response.json();
    return JSON.stringify(data.results);
  }
});

const extractTool = new DynamicStructuredTool({
  name: "extract_content",
  description: "Extract the main content from a webpage URL",
  schema: z.object({
    url: z.string().describe("The URL to extract content from")
  }),
  func: async ({ url }) => {
    const response = await fetch('https://crawlforge.dev/api/v1/tools/extract_content', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ url })
    });

    const data = await response.json();
    return data.content || 'Failed to extract content';
  }
});

// Create the research agent
async function createResearchAgent() {
  const llm = new ChatAnthropic({
    modelName: "claude-3-5-sonnet-20241022"
  });

  const tools = [searchTool, extractTool];

  // Agent will autonomously decide when to search vs extract
  const agent = await createOpenAIFunctionsAgent({
    llm,
    tools,
    prompt: ChatPromptTemplate.fromMessages([
      ["system", "You are a research assistant. Use search_web to find sources, then extract_content to read them in detail."],
      ["human", "{input}"],
      ["placeholder", "{agent_scratchpad}"]
    ])
  });

  return new AgentExecutor({ agent, tools });
}

// Usage
const agent = await createResearchAgent();
const result = await agent.invoke({
  input: "Research the current state of WebAssembly for AI inference"
});

interface CompetitorData {
  name: string;
  pricing: string[];
  features: string[];
  lastUpdated: Date;
}

async function analyzeCompetitor(url: string): Promise<CompetitorData> {
  // Use structured scraping (2 credits)
  const scrapeResponse = await fetch('https://crawlforge.dev/api/v1/tools/scrape_structured', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      url,
      selectors: {
        name: 'h1, .company-name, [data-company]',
        pricing: '.pricing-tier, .plan-price',
        features: '.feature-list li, .features li'
      }
    })
  });

  const scraped = await scrapeResponse.json();

  // Use LLM to structure the data
  const llm = new ChatAnthropic({ modelName: "claude-3-5-sonnet-20241022" });

  const structured = await llm.invoke([
    {
      role: "system",
      content: "Extract competitor data into structured JSON format."
    },
    {
      role: "user",
      content: `Analyze this competitor data: ${JSON.stringify(scraped)}`
    }
  ]);

  return JSON.parse(structured.content as string);
}

// Monitor multiple competitors
async function competitiveIntelligence(competitors: string[]) {
  const results = await Promise.all(
    competitors.map(url => analyzeCompetitor(url))
  );

  // Compare and summarize
  const llm = new ChatAnthropic({ modelName: "claude-3-5-sonnet-20241022" });

  const comparison = await llm.invoke([
    {
      role: "user",
      content: `Compare these competitors and identify market gaps: ${JSON.stringify(results)}`
    }
  ]);

  return { competitors: results, analysis: comparison.content };
}

async function processDocuments(documentUrls: string[]) {
  const documents = [];

  for (const url of documentUrls) {
    // Process document (2 credits)
    const response = await fetch('https://crawlforge.dev/api/v1/tools/process_document', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        source: url,
        sourceType: url.endsWith('.pdf') ? 'pdf_url' : 'url'
      })
    });

    const data = await response.json();
    documents.push({
      url,
      content: data.content,
      metadata: data.metadata
    });
  }

  // Split and index documents
  const splitter = new RecursiveCharacterTextSplitter({
    chunkSize: 1500,
    chunkOverlap: 300
  });

  const chunks = await splitter.createDocuments(
    documents.map(d => d.content),
    documents.map(d => ({ source: d.url, ...d.metadata }))
  );

  return chunks;
}

// Use with LangChain vector store
const chunks = await processDocuments([
  'https://example.com/whitepaper.pdf',
  'https://example.com/technical-spec.pdf'
]);

const vectorStore = await MemoryVectorStore.fromDocuments(
  chunks,
  new OpenAIEmbeddings()
);

import { RunnableSequence } from "@langchain/core/runnables";

async function checkForChanges(url: string, lastHash: string) {
  const response = await fetch('https://crawlforge.dev/api/v1/tools/track_changes', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      url,
      operation: 'compare'
    })
  });

  return response.json();
}

// Create a monitoring chain
const monitoringChain = RunnableSequence.from([
  // Check for changes
  async (input: { url: string }) => {
    const changes = await checkForChanges(input.url, '');
    return { ...input, changes };
  },
  // Analyze changes if found
  async (input) => {
    if (!input.changes.hasChanges) {
      return { ...input, analysis: "No changes detected" };
    }

    const llm = new ChatAnthropic({ modelName: "claude-3-5-sonnet-20241022" });
    const analysis = await llm.invoke([
      {
        role: "user",
        content: `Analyze these website changes: ${JSON.stringify(input.changes)}`
      }
    ]);

    return { ...input, analysis: analysis.content };
  },
  // Generate alert if significant
  async (input) => {
    if (input.analysis === "No changes detected") {
      return null;
    }

    return {
      url: input.url,
      timestamp: new Date(),
      analysis: input.analysis,
      raw: input.changes
    };
  }
]);

// Usage
const alert = await monitoringChain.invoke({
  url: "https://competitor.com/pricing"
});

if (alert) {
  console.log("Change detected:", alert);
}

const cache = new Map<string, { content: string; timestamp: number }>();

async function cachedFetch(url: string, ttl = 3600000) {
  const cached = cache.get(url);
  if (cached && Date.now() - cached.timestamp < ttl) {
    return cached.content;
  }

  const content = await fetchWebContent(url);
  cache.set(url, { content, timestamp: Date.now() });
  return content;
}

// Instead of multiple extract_content calls
const urls = ['url1', 'url2', 'url3'];

// Use batch_scrape (1 credit per URL vs 2 per URL)
const response = await fetch('https://crawlforge.dev/api/v1/tools/batch_scrape', {
  method: 'POST',
  headers: {
    'Authorization': `Bearer ${process.env.CRAWLFORGE_API_KEY}`,
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({ urls })
});

async function fetchWithRetry(url: string, retries = 3) {
  for (let i = 0; i < retries; i++) {
    const response = await fetch(url);

    if (response.status === 429) {
      const retryAfter = response.headers.get('Retry-After') || 60;
      await new Promise(r => setTimeout(r, Number(retryAfter) * 1000));
      continue;
    }

    return response;
  }
  throw new Error('Max retries exceeded');
}