// blurple lab · Are hotel booking sites EAA-ready?
//
// Automated part of the study. For every site in sites.csv it opens up to three
// pages (home, a room or offer page, the first booking step) in a real browser
// on desktop and on a phone, and records:
//   - axe-core violations against WCAG 2.0/2.1/2.2 A and AA, by rule and impact
//   - the page language, horizontal scrolling at 320 px (reflow)
//   - whether an accessibility statement is linked
//   - accessibility overlay widgets on the page
//   - where the "Book" button leads (own site or a booking engine's domain)
//   - an approximate page weight (from content-length headers)
//
// Polite by design: one site at a time, a pause between pages, robots.txt
// respected, a clear user agent, no forms submitted, nothing booked.
//
//   npm install && npx playwright install chromium
//   node scan.mjs --check        reachability of every URL first (fill in fixes)
//   node scan.mjs                full scan → results/<id>.json
//   node scan.mjs --only 7,12    just these site ids
//
// Results are per site; summarize.mjs turns them into the study's tables.

import fs from "node:fs";
import path from "node:path";
import { createRequire } from "node:module";
import { chromium } from "playwright";

const require = createRequire(import.meta.url);
const AXE = fs.readFileSync(require.resolve("axe-core/axe.min.js"), "utf8");
const UA_SUFFIX = " blurple-lab-research/1.0 (+https://blurple.digital/lab)";
const OUT = path.resolve("results");
const PAUSE_MS = 2500;

const args = process.argv.slice(2);
const CHECK_ONLY = args.includes("--check");
const ONLY = args.includes("--only") ? new Set(args[args.indexOf("--only") + 1].split(",").map((x) => x.trim())) : null;
const CSV = args.includes("--sites") ? args[args.indexOf("--sites") + 1] : "sites.csv";

// ── sites.csv: id,name,segment,country,url,room_url,booking_url,verified,notes ──
function readCsv(file) {
  const [head, ...rows] = fs.readFileSync(file, "utf8").trim().split(/\r?\n/);
  const keys = head.split(",");
  return rows
    .filter((r) => r.trim() && !r.startsWith("#"))
    .map((r) => {
      const cells = r.match(/("([^"]|"")*"|[^,]*)(,|$)/g).map((c) => c.replace(/,$/, "").replace(/^"|"$/g, "").replace(/""/g, '"'));
      return Object.fromEntries(keys.map((k, i) => [k, (cells[i] ?? "").trim()]));
    });
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// robots.txt: skip a path that's disallowed for every agent
const robotsCache = new Map();
async function allowedByRobots(url) {
  const u = new URL(url);
  if (!robotsCache.has(u.origin)) {
    let rules = [];
    try {
      const res = await fetch(`${u.origin}/robots.txt`, { signal: AbortSignal.timeout(8000) });
      if (res.ok) {
        let applies = false;
        for (const line of (await res.text()).split(/\r?\n/)) {
          const [k, ...v] = line.split(":");
          const key = k.trim().toLowerCase();
          const val = v.join(":").trim();
          if (key === "user-agent") applies = val === "*";
          else if (applies && key === "disallow" && val) rules.push(val);
        }
      }
    } catch {
      /* no robots.txt: allowed */
    }
    robotsCache.set(u.origin, rules);
  }
  // robots.txt patterns: * matches anything, a trailing $ anchors the end
  const target = u.pathname + u.search;
  return !robotsCache.get(u.origin).some((rule) => {
    const re = new RegExp("^" + rule.replace(/[.+?^{}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$/, "$"));
    return re.test(target);
  });
}

const OVERLAYS = {
  accessiBe: /acsbapp\.com|accessibe\.com/i,
  UserWay: /userway\.org/i,
  EqualWeb: /equalweb\.com/i,
  AudioEye: /audioeye\.com/i,
  "Recite Me": /reciteme\.com/i,
  "Level Access (UsableNet)": /usablenet\.com/i,
};
const STATEMENT = /accessib|erişilebilir|erisilebilir|accesibil|barrierefrei|barrierefreiheit|accessibilit|toegankelijk/i;
const BOOK = /\b(book|booking|reserve|reservation|check availability|rezervasyon|rezerve|reserva|reservar|buchen|réserver)\b/i;

async function scanPage(context, url, viewport) {
  const page = await context.newPage();
  const requests = new Set();
  let bytes = 0;
  page.on("request", (r) => requests.add(r.url()));
  page.on("response", async (r) => {
    const len = Number(r.headers()["content-length"] || 0);
    bytes += len;
  });
  const out = { url, viewport: `${viewport.width}x${viewport.height}` };
  try {
    const res = await page.goto(url, { waitUntil: "load", timeout: 45000 });
    out.status = res?.status() ?? 0;
    out.finalUrl = page.url();
    await page.waitForTimeout(2500);
    // close nothing, accept nothing: we measure the page as a first-time visitor meets it
    // a bot wall (Cloudflare, Akamai, …) instead of the site: record it, measure nothing
    const wall = await page.evaluate(() => {
      const t = `${document.title} ${document.body?.innerText.slice(0, 600) ?? ""}`;
      return /just a moment|performing security verification|verify you are human|attention required|access denied|pardon our interruption|are you a robot|request unsuccessful|incapsula/i.test(t) || document.querySelectorAll("a[href]").length < 3;
    });
    if (wall) {
      out.blocked = true;
      await page.close();
      return out;
    }
    out.lang = await page.evaluate(() => document.documentElement.getAttribute("lang") || "");
    out.reflowOverflow =
      viewport.width <= 390
        ? await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth + 1)
        : null;
    out.statementLink = await page.evaluate(
      (re) => [...document.querySelectorAll("a[href]")].some((a) => new RegExp(re, "i").test(`${a.textContent} ${a.getAttribute("href")}`)),
      STATEMENT.source,
    );
    out.bookingLinks = await page.evaluate(
      (re) =>
        [...new Set([...document.querySelectorAll("a[href], button, form[action]")]
          .filter((el) => new RegExp(re, "i").test(el.textContent || el.getAttribute("aria-label") || ""))
          .map((el) => {
            const href = el.getAttribute("href") || el.getAttribute("action") || "";
            try { return new URL(href, location.href).hostname; } catch { return ""; }
          })
          .filter(Boolean))].slice(0, 5),
      BOOK.source,
    );
    await page.addScriptTag({ content: AXE });
    const axe = await page.evaluate(async () => {
      const r = await window.axe.run(document, { runOnly: { type: "tag", values: ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22aa"] }, resultTypes: ["violations"] });
      return r.violations.map((v) => ({ id: v.id, impact: v.impact, tags: v.tags.filter((t) => /^wcag\d/.test(t)), nodes: v.nodes.length }));
    });
    out.violations = axe;
    out.overlays = Object.entries(OVERLAYS).filter(([, re]) => [...requests].some((u) => re.test(u))).map(([k]) => k);
    // from content-length headers: a floor, not an exact weight (chunked responses report none)
    out.approxBytes = bytes;
  } catch (e) {
    out.error = e.message.split("\n")[0].slice(0, 200);
  }
  await page.close();
  return out;
}

// our clients and partners are left out of sites.csv itself
const sites = readCsv(CSV).filter((s) => !ONLY || ONLY.has(s.id));
fs.mkdirSync(OUT, { recursive: true });
const browser = await chromium.launch({
  ...(process.env.CHROMIUM_PATH ? { executablePath: process.env.CHROMIUM_PATH } : {}),
});
const probe = await browser.newPage();
const ua = (await probe.evaluate(() => navigator.userAgent)).replace("HeadlessChrome", "Chrome") + UA_SUFFIX;
await probe.close();
const MOBILE_UA = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1" + UA_SUFFIX;

if (CHECK_ONLY) {
  const rows = ["id,name,url,status,final_url,robots_ok"];
  for (const s of sites) {
    let status = 0, finalUrl = "", ok = "";
    try {
      ok = (await allowedByRobots(s.url)) ? "yes" : "no";
      const res = await fetch(s.url, { redirect: "follow", headers: { "user-agent": ua }, signal: AbortSignal.timeout(15000) });
      status = res.status;
      finalUrl = res.url;
    } catch (e) {
      finalUrl = `ERROR ${e.message.slice(0, 80)}`;
    }
    rows.push([s.id, `"${s.name}"`, s.url, status, finalUrl, ok].join(","));
    console.log(s.id, status, s.name, "→", finalUrl);
    await sleep(500);
  }
  fs.writeFileSync(path.join(OUT, "check.csv"), rows.join("\n") + "\n");
  await browser.close();
  process.exit(0);
}

for (const s of sites) {
  const file = path.join(OUT, `${s.id}.json`);
  // finished sites are kept; --force scans them again
  if (fs.existsSync(file) && !args.includes("--force")) {
    console.log(s.id, "already scanned, skipping");
    continue;
  }
  const locale = s.country === "TR" ? "tr-TR" : "en-GB";
  // desktop and phone are separate browser profiles, so sites serve their real mobile version
  const contexts = {
    "1440x900": await browser.newContext({ userAgent: ua, locale, viewport: { width: 1440, height: 900 } }),
    "390x844": await browser.newContext({ userAgent: MOBILE_UA, locale, viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true, deviceScaleFactor: 3 }),
  };
  const pages = [
    ["home", s.url],
    ["room", s.room_url],
    ["booking", s.booking_url],
  ].filter(([, u]) => u);
  const result = { id: s.id, name: s.name, segment: s.segment, country: s.country, scannedAt: new Date().toISOString(), pages: [] };
  for (const [kind, url] of pages) {
    if (!(await allowedByRobots(url))) {
      result.pages.push({ kind, url, skipped: "robots.txt" });
      continue;
    }
    for (const viewport of [{ width: 1440, height: 900 }, { width: 390, height: 844 }]) {
      const r = await scanPage(contexts[`${viewport.width}x${viewport.height}`], url, viewport);
      result.pages.push({ kind, ...r });
      console.log(s.id, kind, r.viewport, r.error ? `ERROR ${r.error}` : r.blocked ? "BLOCKED (bot protection)" : `${r.violations.length} rules failed`);
      await sleep(PAUSE_MS);
    }
  }
  for (const c of Object.values(contexts)) await c.close();
  fs.writeFileSync(file, JSON.stringify(result, null, 2));
}
await browser.close();
console.log(`done: ${sites.length} sites → ${OUT}`);