Web Context: Decide What a Fetched Page Is Before You Pay to Read It

const triaged = await Promise.all(
  fetched.map(async (page) => {
    const { answers } = await jev.decide({
      state: [
        `QUERY THAT SENT US HERE\n${query}`,
        `URL\n${page.url}`,
        `FETCHED TEXT (truncated)\n${page.text.slice(0, 8000)}`,
      ].join("\n\n"),
      questions: {
        answers_query: {
          type: "noul",
          instructions:
            "Does this page contain the information needed to answer the query, even if it is not the main subject of the page?",
        },
        page_type: {
          type: "choice",
          instructions: "Select what kind of page this is.",
          criteria: {
            documentation:   "Reference material, specifications or manuals.",
            product_page:    "One purchasable product, with its details.",
            editorial:       "A guide, blog post or review.",
            listing:         "A category or search page that mostly links elsewhere.",
            blocked_or_empty:"Consent wall, paywall, error, or no usable content.",
          },
        },
        context_value: {
          type: "score",
          instructions:
            "Rate how much of this page is worth putting into a model context window for this query.",
          criteria: [
            "None - drop it",
            "One or two useful sentences buried in boilerplate",
            "A useful section worth extracting",
            "Substantially on-topic - keep the page",
          ],
        },
      },
    });
    return { page, ...answers };
  })
);

// Three branches, one score.
const keep    = triaged.filter((t) => t.context_value.score >= 2.5);
const extract = triaged.filter((t) => t.context_value.score >= 1 && t.context_value.score < 2.5);
const retry   = triaged.filter((t) => t.page_type.choice === "blocked_or_empty");