openapi: 3.0.3
info:
  title: YakSpider API
  version: "1.0.0"
  description: >
    Web scraping, crawling and discovery API. Submit ad-hoc scrapes or drive
    configured spiders (site-crawl or discovery), then read runs, results, and
    the discovered-artifact catalog.
servers:
  - url: /api/v1
security:
  - ApiKeyAuth: []
components:
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-API-Key
  schemas:
    Error:
      type: object
      properties:
        errors:
          type: object
          properties:
            detail: { type: string }
            code: { type: string, description: "Stable error code (e.g. credit_exhausted, tier_not_allowed); present on plan-limit errors." }
            doc: { type: string, description: "Path to the docs anchor explaining this error." }
    Spider:
      type: object
      properties:
        id: { type: integer }
        name: { type: string }
        slug: { type: string }
        mode: { type: string, enum: [site_crawl, discovery] }
        status: { type: string }
        record_type: { type: string, nullable: true }
        tier: { type: string, enum: [t1, t2, t3, t4, t5] }
        escalate: { type: boolean }
        obey_robots: { type: boolean }
        rate_limit_rps: { type: number }
        seed_urls: { type: array, items: { type: string } }
        scope: { type: object }
    Run:
      type: object
      properties:
        id: { type: integer }
        spider_id: { type: integer }
        mode: { type: string }
        status: { type: string, enum: [queued, running, succeeded, failed, partial, canceled] }
        tier_used: { type: string, nullable: true }
        pages_crawled: { type: integer }
        results_count: { type: integer }
    CatalogEntry:
      type: object
      properties:
        id: { type: integer }
        record_type: { type: string }
        url: { type: string }
        status: { type: string }
        metadata: { type: object }
    ScrapeResult:
      type: object
      properties:
        data:
          type: object
          properties:
            config:
              type: object
              description: Echoes the request (url, tier, render, format).
            context: { type: object }
            result:
              type: object
              properties:
                status_code: { type: integer }
                final_url: { type: string }
                content_type: { type: string, nullable: true }
                content:
                  type: string
                  description: Page body rendered in the requested `format` (html/text/markdown).
                extracted: { type: array, items: { type: object } }
paths:
  /scrape:
    post:
      summary: Submit a scrape (sync single-URL, or async by spider_id)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              oneOf:
                - type: object
                  required: [spider_id]
                  properties:
                    spider_id: { type: integer }
                - type: object
                  required: [url]
                  properties:
                    url: { type: string }
                    tier: { type: string, enum: [t1, t2] }
                    escalate: { type: boolean }
                    format:
                      type: string
                      enum: [html, text, markdown]
                      default: html
                      description: Output format for `result.content`. `markdown` returns LLM-ready Markdown.
                    session:
                      type: string
                      description: Named session — stored cookies are sent and any set-cookie folds back in.
                    extract:
                      type: array
                      items:
                        type: object
                        properties:
                          name: { type: string }
                          strategy: { type: string, enum: [css, json, regex] }
                          selector: { type: string }
                          attr: { type: string }
                          data_type: { type: string }
                          many: { type: boolean }
      responses:
        "200":
          description: Synchronous scrape result
          content:
            application/json:
              schema: { $ref: "#/components/schemas/ScrapeResult" }
        "202":
          description: Async run started
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: object
                    properties:
                      run_id: { type: integer }
                      status: { type: string }
        "401": { description: Invalid API key, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "402": { description: "Plan limit: monthly credit allowance exhausted (code credit_exhausted) or billing inactive (feature_inactive)", content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "403": { description: "Requested tier is not included in your plan (code tier_not_allowed)", content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "422": { description: Invalid request, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "429": { description: "Concurrent run limit reached for your plan (code concurrency_exceeded)", content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
  /scrape/{id}:
    get:
      summary: Get a run's status and results
      parameters:
        - { name: id, in: path, required: true, schema: { type: integer } }
      responses:
        "200": { description: Run envelope, content: { application/json: { schema: { $ref: "#/components/schemas/ScrapeResult" } } } }
        "404": { description: Not found, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
  /batch:
    post:
      summary: Batch scrape — fetch many URLs in one async ad-hoc run (no saved spider)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required: [urls]
              properties:
                urls: { type: array, items: { type: string }, maxItems: 1000 }
                tier: { type: string, enum: [t1, t2, t3, t4, t5] }
                escalate: { type: boolean }
                extract:
                  type: array
                  items:
                    type: object
                    properties:
                      name: { type: string }
                      strategy: { type: string, enum: [css, json, regex] }
                      selector: { type: string }
                      attr: { type: string }
      responses:
        "202":
          description: Batch run started — poll GET /scrape/{id} for results
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: object
                    properties:
                      run_id: { type: integer }
                      status: { type: string }
                      urls: { type: integer }
        "402": { description: Credits exhausted / inactive, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "403": { description: Tier not in plan, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "422": { description: Invalid request, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
  /screenshot:
    post:
      summary: Render a URL with the headless browser and return a base64 image
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required: [url]
              properties:
                url: { type: string }
                format: { type: string, enum: [png, jpeg], default: png }
                full_page: { type: boolean }
      responses:
        "200":
          description: Screenshot captured
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: object
                    properties:
                      config: { type: object }
                      result:
                        type: object
                        properties:
                          status_code: { type: integer }
                          image_b64: { type: string }
        "402": { description: Credits exhausted / inactive, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "403": { description: Tier not in plan, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
        "503": { description: Browser service unavailable, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
  /preview:
    post:
      summary: Dry-run a spider config against live conditions (no persistence)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              oneOf:
                - type: object
                  required: [spider_id]
                  properties:
                    spider_id: { type: integer }
                - type: object
                  properties:
                    name: { type: string }
                    mode: { type: string, enum: [site_crawl, discovery] }
                    tier: { type: string }
                    seed_urls: { type: array, items: { type: string } }
                    scope: { type: object }
                    extraction_rules: { type: array, items: { type: object } }
                    sources: { type: array, items: { type: object } }
      responses:
        "200":
          description: Preview feedback (per-target status + warnings)
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: object
                    properties:
                      ok?: { type: boolean }
                      mode: { type: string }
                      targets: { type: array, items: { type: object } }
                      warnings: { type: array, items: { type: string } }
  /spiders:
    get:
      summary: List spiders
      responses:
        "200":
          description: Spiders
          content:
            application/json:
              schema:
                type: object
                properties:
                  data: { type: array, items: { $ref: "#/components/schemas/Spider" } }
    post:
      summary: Create a spider
      requestBody:
        required: true
        content:
          application/json:
            schema: { $ref: "#/components/schemas/Spider" }
      responses:
        "201": { description: Created, content: { application/json: { schema: { type: object, properties: { data: { $ref: "#/components/schemas/Spider" } } } } } }
        "422": { description: Invalid, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } }
  /spiders/{id}:
    get:
      summary: Get a spider
      parameters: [{ name: id, in: path, required: true, schema: { type: integer } }]
      responses:
        "200": { description: Spider, content: { application/json: { schema: { type: object, properties: { data: { $ref: "#/components/schemas/Spider" } } } } } }
        "404": { description: Not found }
    patch:
      summary: Update a spider
      parameters: [{ name: id, in: path, required: true, schema: { type: integer } }]
      requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/Spider" } } } }
      responses:
        "200": { description: Updated }
        "422": { description: Invalid }
    delete:
      summary: Delete a spider
      parameters: [{ name: id, in: path, required: true, schema: { type: integer } }]
      responses:
        "204": { description: Deleted }
  /spiders/{id}/runs:
    get:
      summary: List a spider's runs
      parameters: [{ name: id, in: path, required: true, schema: { type: integer } }]
      responses:
        "200":
          description: Runs
          content:
            application/json:
              schema:
                type: object
                properties:
                  data: { type: array, items: { $ref: "#/components/schemas/Run" } }
  /catalog:
    get:
      summary: Query the discovered-artifact catalog
      parameters:
        - { name: record_type, in: query, required: false, schema: { type: string } }
        - { name: limit, in: query, required: false, schema: { type: integer } }
      responses:
        "200":
          description: Catalog entries
          content:
            application/json:
              schema:
                type: object
                properties:
                  data: { type: array, items: { $ref: "#/components/schemas/CatalogEntry" } }
                  meta: { type: object, properties: { count: { type: integer } } }
