How to Build a RAG Pipeline with Web Data

import { CrawlForge } from '@crawlforge/sdk';

const cf = new CrawlForge({ apiKey: process.env.CRAWLFORGE_API_KEY });

// Crawl a documentation site up to 3 levels deep
const crawlResult = await cf.crawlDeep({
  url: 'https://docs.example.com',
  max_depth: 3,
  max_pages: 200,
  include_patterns: ['/docs/', '/guides/', '/api/'],
  exclude_patterns: ['/changelog', '/blog'],
  extract_content: true, // Get clean text during crawl
  concurrency: 10
});

console.log(`Crawled ${crawlResult.pages.length} pages`);
// Crawled 147 pages

// Scrape a specific list of URLs in parallel
const batchResult = await cf.batchScrape({
  urls: [
    'https://docs.example.com/auth',
    'https://docs.example.com/billing',
    'https://docs.example.com/webhooks',
    'https://docs.example.com/rate-limits',
    // ... up to 50 URLs per batch
  ],
  formats: ['text'],
  maxConcurrency: 10
});

// Extract clean content from a single page
const page = await cf.extractContent({
  url: 'https://docs.example.com/authentication'
});

console.log(page.content);
// Returns: "Authentication

All API requests require..."
// No nav bars, no footers, no cookie banners

interface TextChunk {
  content: string;
  metadata: {
    source: string;
    title: string;
    heading: string;
    chunkIndex: number;
  };
}

function chunkByHeading(
  text: string,
  source: string,
  title: string,
  maxChunkSize: number = 1000 // characters
): TextChunk[] {
  const chunks: TextChunk[] = [];
  // Split on markdown headings (## or ###)
  const sections = text.split(/(?=^#{2,3}\s)/m);

  let chunkIndex = 0;
  for (const section of sections) {
    // Extract heading from section
    const headingMatch = section.match(/^#{2,3}\s+(.+)/);
    const heading = headingMatch ? headingMatch[1].trim() : title;
    const content = section.trim();

    if (content.length <= maxChunkSize) {
      chunks.push({
        content,
        metadata: { source, title, heading, chunkIndex: chunkIndex++ }
      });
    } else {
      // Split large sections by paragraph
      const paragraphs = content.split('\n\n');
      let currentChunk = '';

      for (const para of paragraphs) {
        if ((currentChunk + para).length > maxChunkSize && currentChunk) {
          chunks.push({
            content: currentChunk.trim(),
            metadata: { source, title, heading, chunkIndex: chunkIndex++ }
          });
          currentChunk = para;
        } else {
          currentChunk += (currentChunk ? '\n\n' : '') + para;
        }
      }
      if (currentChunk.trim()) {
        chunks.push({
          content: currentChunk.trim(),
          metadata: { source, title, heading, chunkIndex: chunkIndex++ }
        });
      }
    }
  }

  return chunks;
}

// Usage with crawled pages
const allChunks: TextChunk[] = [];
for (const page of crawlResult.pages) {
  const pageChunks = chunkByHeading(
    page.content,
    page.url,
    page.title || 'Untitled',
    1000
  );
  allChunks.push(...pageChunks);
}

console.log(`Created ${allChunks.length} chunks from ${crawlResult.pages.length} pages`);
// Created 892 chunks from 147 pages

import OpenAI from 'openai';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

interface EmbeddedChunk extends TextChunk {
  embedding: number[];
}

async function embedChunks(
  chunks: TextChunk[],
  batchSize: number = 100
): Promise<EmbeddedChunk[]> {
  const embeddedChunks: EmbeddedChunk[] = [];

  for (let i = 0; i < chunks.length; i += batchSize) {
    const batch = chunks.slice(i, i + batchSize);
    const texts = batch.map(c => c.content);

    const response = await openai.embeddings.create({
      model: 'text-embedding-3-small', // 1536 dimensions, $0.02/1M tokens
      input: texts
    });

    for (let j = 0; j < batch.length; j++) {
      embeddedChunks.push({
        ...batch[j],
        embedding: response.data[j].embedding
      });
    }

    console.log(`Embedded ${Math.min(i + batchSize, chunks.length)}/${chunks.length} chunks`);
  }

  return embeddedChunks;
}

const embeddedChunks = await embedChunks(allChunks);

import { Pinecone } from '@pinecone-database/pinecone';

const pinecone = new Pinecone({ apiKey: process.env.PINECONE_API_KEY });
const index = pinecone.Index('rag-web-data');

// Upsert chunks in batches of 100
async function storeChunks(chunks: EmbeddedChunk[]) {
  const batchSize = 100;

  for (let i = 0; i < chunks.length; i += batchSize) {
    const batch = chunks.slice(i, i + batchSize);
    const vectors = batch.map((chunk, j) => ({
      id: `${chunk.metadata.source}-${chunk.metadata.chunkIndex}`,
      values: chunk.embedding,
      metadata: {
        content: chunk.content,
        source: chunk.metadata.source,
        title: chunk.metadata.title,
        heading: chunk.metadata.heading
      }
    }));

    await index.upsert(vectors);
    console.log(`Stored ${Math.min(i + batchSize, chunks.length)}/${chunks.length} vectors`);
  }
}

await storeChunks(embeddedChunks);

import weaviate from 'weaviate-ts-client';

const client = weaviate.client({
  scheme: 'https',
  host: process.env.WEAVIATE_HOST || 'localhost:8080'
});

// Create collection schema
await client.schema.classCreator().withClass({
  class: 'WebDocument',
  vectorizer: 'none', // We provide our own vectors
  properties: [
    { name: 'content', dataType: ['text'] },
    { name: 'source', dataType: ['string'] },
    { name: 'title', dataType: ['string'] },
    { name: 'heading', dataType: ['string'] }
  ]
}).do();

// Store chunks
for (const chunk of embeddedChunks) {
  await client.data.creator()
    .withClassName('WebDocument')
    .withProperties({
      content: chunk.content,
      source: chunk.metadata.source,
      title: chunk.metadata.title,
      heading: chunk.metadata.heading
    })
    .withVector(chunk.embedding)
    .do();
}

async function ragQuery(question: string): Promise<string> {
  // 1. Embed the question
  const questionEmbedding = await openai.embeddings.create({
    model: 'text-embedding-3-small',
    input: question
  });

  // 2. Query Pinecone for the 5 most relevant chunks
  const queryResult = await index.query({
    vector: questionEmbedding.data[0].embedding,
    topK: 5,
    includeMetadata: true
  });

  // 3. Build context from retrieved chunks
  const context = queryResult.matches
    .map(match => {
      const meta = match.metadata as Record<string, string>;
      return `Source: ${meta.source}\nSection: ${meta.heading}\n\n${meta.content}`;
    })
    .join('\n\n---\n\n');

  // 4. Generate answer with context
  const completion = await openai.chat.completions.create({
    model: 'gpt-4o',
    messages: [
      {
        role: 'system',
        content: `Answer the user's question based on the provided context.
Cite sources when possible. If the context does not contain enough
information, say so.`
      },
      {
        role: 'user',
        content: `Context:\n${context}\n\nQuestion: ${question}`
      }
    ]
  });

  return completion.choices[0].message.content || 'No answer generated';
}

// Example usage
const answer = await ragQuery('How does authentication work?');
console.log(answer);

async function buildRAGPipeline(config: {
  targetUrl: string;
  maxPages: number;
  indexName: string;
}) {
  const cf = new CrawlForge({ apiKey: process.env.CRAWLFORGE_API_KEY });

  // 1. Crawl
  console.log('Crawling...');
  const crawled = await cf.crawlDeep({
    url: config.targetUrl,
    max_depth: 3,
    max_pages: config.maxPages,
    extract_content: true,
    concurrency: 10
  });
  console.log(`Crawled ${crawled.pages.length} pages`);

  // 2. Chunk
  console.log('Chunking...');
  const chunks = crawled.pages.flatMap(page =>
    chunkByHeading(page.content, page.url, page.title || 'Untitled')
  );
  console.log(`Created ${chunks.length} chunks`);

  // 3. Embed
  console.log('Embedding...');
  const embedded = await embedChunks(chunks);

  // 4. Store
  console.log('Storing...');
  await storeChunks(embedded);

  console.log('RAG pipeline complete. Ready for queries.');
}

// Run it
await buildRAGPipeline({
  targetUrl: 'https://docs.example.com',
  maxPages: 200,
  indexName: 'rag-web-data'
});