Firecrawl

Copy codeconst firecrawl = new FirecrawlClient(components.firecrawl);

// One-shot
const page = await firecrawl.scrape(ctx, "https://firecrawl.dev", {
  formats: ["markdown"],
});

// Durable: returns immediately, pages stream into your database
const { crawlId } = await firecrawl.startCrawl(ctx, {
  url: "https://docs.firecrawl.dev",
  options: { limit: 50 },
  onComplete: internal.myModule.indexCrawledPages,
});

Copy codenpm install @firecrawl/firecrawl-convex

Copy code// convex/convex.config.ts
import { defineApp } from "convex/server";
import { v } from "convex/values";
import firecrawl from "@firecrawl/firecrawl-convex/convex.config";

const app = defineApp({
  env: {
    FIRECRAWL_API_KEY: v.string(),
    FIRECRAWL_WEBHOOK_SECRET: v.optional(v.string()),
  },
});

app.use(firecrawl, {
  // Mounts the webhook route at <your-site>/firecrawl/webhook.
  // Required for crawls in webhook mode.
  httpPrefix: "/firecrawl/",
  env: {
    FIRECRAWL_API_KEY: app.env.FIRECRAWL_API_KEY,
    FIRECRAWL_WEBHOOK_SECRET: app.env.FIRECRAWL_WEBHOOK_SECRET,
  },
});

export default app;

Copy codenpx convex env set FIRECRAWL_API_KEY fc-your-key
# Recommended: from the Firecrawl dashboard → Advanced → webhook secret
npx convex env set FIRECRAWL_WEBHOOK_SECRET whsec-your-secret
npx convex dev

Copy code// convex/web.ts
import { v } from "convex/values";
import { FirecrawlClient } from "@firecrawl/firecrawl-convex";
import { action } from "./_generated/server";
import { components } from "./_generated/api";

const firecrawl = new FirecrawlClient(components.firecrawl);

export const scrapePage = action({
  args: { url: v.string() },
  handler: async (ctx, args) => {
    await requireUser(ctx);
    return await firecrawl.scrape(ctx, args.url, {
      formats: ["markdown", { type: "json", prompt: "Extract the pricing table" }],
      onlyMainContent: true,
      maxAge: 3_600_000, // reuse Firecrawl's cache for an hour
    });
  },
});

export const siteUrls = action({
  args: { url: v.string() },
  handler: (ctx, args) => firecrawl.map(ctx, args.url, { limit: 500 }),
});

export const searchWeb = action({
  args: { query: v.string() },
  handler: (ctx, args) =>
    firecrawl.search(ctx, args.query, {
      limit: 5,
      scrapeOptions: { formats: ["markdown"] },
    }),
});

Copy codeawait firecrawl.scrape(ctx, url, { extra: { threatProtection: { mode: "off" } } });

Copy codeexport const crawlDocs = action({
  args: { url: v.string() },
  handler: async (ctx, args) => {
    const userId = await requireUser(ctx);
    return await firecrawl.startCrawl(ctx, {
      url: args.url,
      options: {
        limit: 100,
        includePaths: ["^/docs/.*"],
        scrapeOptions: { formats: ["markdown"], onlyMainContent: true },
      },
      onComplete: internal.web.onCrawlComplete,
      context: { userId },
    });
  },
});

Copy code// Live status: total, completed, pageCount, creditsUsed, error
export const crawlProgress = query({
  args: { crawlId: v.string() },
  handler: (ctx, args) => firecrawl.getCrawl(ctx, args.crawlId),
});

// Pages as they land — works with usePaginatedQuery
export const crawlPages = query({
  args: { crawlId: v.string(), paginationOpts: paginationOptsValidator },
  handler: (ctx, args) => firecrawl.listPages(ctx, args),
});

Copy codefunction CrawlView({ crawlId }: { crawlId: string }) {
  const crawl = useQuery(api.web.crawlProgress, { crawlId });
  const { results } = usePaginatedQuery(
    api.web.crawlPages,
    { crawlId },
    { initialNumItems: 25 },
  );
  return (
    <>
      <progress value={crawl?.pageCount ?? 0} max={crawl?.total ?? 1} />
      <ul>{results.map((p) => <li key={p._id}>{p.url}</li>)}</ul>
    </>
  );
}

Copy codeexport const onCrawlComplete = internalMutation({
  args: {
    crawlId: v.string(),
    jobId: v.optional(v.string()),
    status: v.union(v.literal("completed"), v.literal("failed"), v.literal("cancelled")),
    pageCount: v.number(),
    unstored: v.optional(v.number()),
    error: v.optional(v.string()),
    context: v.optional(v.any()),
  },
  handler: async (ctx, args) => {
    if (args.status !== "completed") return;
    if (args.unstored) console.warn(`${args.unstored} pages were too large to store`);
    // e.g. hand the pages to an embedding pipeline
    await ctx.scheduler.runAfter(0, internal.rag.indexCrawl, {
      crawlId: args.crawlId,
      userId: args.context?.userId,
    });
  },
});

Copy codeawait firecrawl.startCrawl(ctx, { url, mode: "poll" });

Copy codeawait firecrawl.getCrawlByJobId(ctx, jobId);       // look up by Firecrawl's id
await firecrawl.listCrawls(ctx, { status: "scraping", limit: 20 });
await firecrawl.getPage(ctx, { crawlId, url });
await firecrawl.cancelCrawl(ctx, crawlId);         // action
await firecrawl.deleteCrawl(ctx, crawlId);         // mutation: crawl + its pages
await firecrawl.resumeCrawl(ctx, crawlId);         // mutation: see below

Copy codeimport { convexTest } from "convex-test";
import firecrawl from "@firecrawl/firecrawl-convex/test";
import schema from "./schema";

const modules = import.meta.glob("./**/*.*s");

export function initConvexTest() {
  process.env.FIRECRAWL_API_KEY = "fc-test-key";
  const t = convexTest(schema, modules);
  firecrawl.register(t);
  return t;
}

Copy codenpm install

# terminal 1
FIRECRAWL_WEBHOOK_SECRET=whsec-mock npm run dev:mock

# terminal 2
npx convex env set FIRECRAWL_API_KEY fc-mock-key
npx convex env set FIRECRAWL_API_URL http://127.0.0.1:4242
npx convex env set FIRECRAWL_WEBHOOK_SECRET whsec-mock
npm run dev

# terminal 3
npx convex env set DEMO_ALLOW_ANONYMOUS true   # local CLI demo only
npx convex run example:startCrawl '{"url":"https://mock.test","limit":3}'
npx convex run example:myCrawls '{}'
npx convex run example:reports '{}'

Copy codenpm run dev         # component codegen + build watcher + convex dev
npm test            # vitest, including type tests
npm run typecheck
npm run lint