Bright Data

App calls trigger()
        ↓
Bright Data collection job starts
        ↓
Component stores snapshot as "pending"
        ↓
Bright Data POSTs results to webhook handler
        ↓
Component parses NDJSON, stores records, marks snapshot "ready"
        ↓
All useQuery subscribers notified automatically
        ↓
UI updates in real time

Copy codenpm install @sholajegede/convex-bright-data-datasets

Copy codeimport { defineApp } from "convex/server";
import convexBrightDataDatasets from "@sholajegede/convex-bright-data-datasets/convex.config.js";

const app = defineApp();
app.use(convexBrightDataDatasets);

export default app;

Copy code// convex/brightDatasets.ts
import { components } from "./_generated/api.js";
import { BrightDatasets } from "@sholajegede/convex-bright-data-datasets";

export const brightDatasets = new BrightDatasets(components.convexBrightDataDatasets, {
  BRIGHTDATA_API_TOKEN: process.env.BRIGHTDATA_API_TOKEN!,
});

Copy codeimport { httpRouter } from "convex/server";
import { components } from "./_generated/api.js";
import { createWebhookHandler } from "@sholajegede/convex-bright-data-datasets";

const http = httpRouter();

http.route({
  path: "/webhooks/brightdata",
  method: "POST",
  handler: createWebhookHandler(components.convexBrightDataDatasets),
});

export default http;

Copy codenpx convex env set BRIGHTDATA_API_TOKEN your_token_here

https://<your-deployment>.convex.site/webhooks/brightdata

Copy code// convex/myFunctions.ts
import { action, query } from "./_generated/server.js";
import { components } from "./_generated/api.js";
import { brightDatasets } from "./brightDatasets.js";
import { v } from "convex/values";

// Trigger a LinkedIn profile collection
export const collectProfiles = action({
  args: { urls: v.array(v.string()) },
  handler: async (ctx, args) => {
    return await brightDatasets.trigger(ctx, {
      datasetId: "gd_l1viktl72bvl7bjuj0", // LinkedIn profiles dataset
      inputs: args.urls.map((url) => ({ url })),
      webhookUrl: process.env.CONVEX_SITE_URL + "/webhooks/brightdata",
    });
    // Returns: { snapshotId: "s_...", status: "pending" }
  },
});

// Reactive query — subscribe to snapshot status from the frontend
export const getSnapshot = query({
  args: { snapshotId: v.string() },
  handler: async (ctx, args) => {
    return await ctx.runQuery(components.convexBrightDataDatasets.lib.getSnapshot, {
      snapshotId: args.snapshotId,
    });
  },
});

// Reactive query — subscribe to records as they arrive
export const getRecords = query({
  args: { snapshotId: v.string() },
  handler: async (ctx, args) => {
    return await ctx.runQuery(components.convexBrightDataDatasets.lib.getRecords, {
      snapshotId: args.snapshotId,
    });
  },
});

Copy code// React — subscribes reactively, re-renders when status or records update
const snapshot = useQuery(api.myFunctions.getSnapshot, { snapshotId });
// snapshot.status   — "pending" | "collecting" | "digesting" | "ready" | "failed" | "canceled"
// snapshot.recordCount — number of records received so far

const records = useQuery(api.myFunctions.getRecords, { snapshotId });
// records — array of structured records from Bright Data, parsed from NDJSON

Copy codeexport const scrapeProfile = action({
  args: { url: v.string() },
  handler: async (ctx, args) => {
    return await brightDatasets.scrape(ctx, {
      datasetId: "gd_l1viktl72bvl7bjuj0",
      inputs: [{ url: args.url }],
    });
    // Returns: { records: [...], status: "ready" }
    // If job exceeds 1 min: { records: [], snapshotId: "s_...", status: "running" }
  },
});

Copy codeexport const checkStatus = action({
  args: { snapshotId: v.string() },
  handler: async (ctx, args) => {
    return await brightDatasets.pollStatus(ctx, args.snapshotId);
    // Fetches from Bright Data, updates snapshot in Convex, returns current status
  },
});

Copy codeexport const cancelJob = action({
  args: { snapshotId: v.string() },
  handler: async (ctx, args) => {
    return await brightDatasets.cancel(ctx, args.snapshotId);
  },
});

Copy codeexport const listJobs = query({
  args: {},
  handler: async (ctx) => {
    return await ctx.runQuery(components.convexBrightDataDatasets.lib.listSnapshots, {
      limit: 20,
    });
  },
});

Copy code// Discover Amazon products by keyword
export const discoverProducts = action({
  args: { keywords: v.array(v.string()) },
  handler: async (ctx, args) => {
    return await brightDatasets.trigger(ctx, {
      datasetId: "gd_l7q7dkf244hwjntr0",
      inputs: args.keywords.map((keyword) => ({ keyword })),
      discoveryMode: "discover_new",
      discoverBy: "keyword",
      limitPerInput: 10,
      webhookUrl: process.env.CONVEX_SITE_URL + "/webhooks/brightdata",
    });
  },
});

pending → collecting → digesting → ready
                                 → failed
                                 → canceled

Copy codenpm i
npm run dev