id: robots-txt-endpoint

info:
  name: robots.txt endpoint prober
  author: CasperGN,pdteam
  severity: info
  metadata:
    max-request: 2
  tags: miscellaneous,misc,generic

http:
  - method: GET
    path:
      - "{{RootURL}}/robots.txt"
      - "{{RootURL}}{{endpoint}}"

    extractors:
      - type: regex
        name: endpoint
        group: 1
        regex:
          - '(?m:\s(/[[:alpha:]]+[[:graph:]]+))'
        internal: true
    iterate-all: true

    matchers-condition: and
    matchers:
      - type: word
        part: body
        words:
          - 'User-agent:'
          - 'Disallow:'
          - 'Allow:'

      - type: word
        part: header
        words:
          - "text/plain"

      - type: status
        status:
          - 200
# digest: 490a0046304402202c24e04c0569f2d4ccbd4cbf23b95ad24caed4a8945cd5998cae05f4ab36dd8302204017442c36f8070b36207d9070c10230844e496e3fd4ab4af88e5bd985d34a87:922c64590222798bb761d5b6d8e72950