Add the ability to pass EvaluationPlugins to the validate function

Author: jdesrosiers

README.md

@@ -187,6 +187,57 @@ addMediaTypePlugin("application/schema+yaml", {
 });
 ```
 
+**EvaluationPlugins**
+
+EvaluationPlugins allow you to hook into the validation process for various
+purposes. There are hooks for before an after schema evaluation and before and
+after keyword evaluation. (See the API section for the full interface) The
+following is a simple example to record all the schema locations that were
+evaluated. This could be used as part of a solution for determining test
+coverage for a schema.
+
+```JavaScript
+import { registerSchema, validate } from "@hyperjump/json-schema/draft-2020-12";
+import { BASIC } from "@hyperjump/json-schema/experimental.js";
+
+class EvaluatedKeywordsPlugin {
+  constructor() {
+    this.schemaLocations = new Set();
+  }
+
+  beforeKeyword([, schemaUri]) {
+    this.schemaLocations.add(schemaUri);
+  }
+}
+
+registerSchema({
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  type: "object",
+  properties: {
+    foo: { type: "number" },
+    bar: { type: "boolean" }
+  },
+  required: ["foo"]
+}, "https://schemas.hyperjump.io/main");
+
+const evaluatedKeywordPlugin = new EvaluatedKeywordsPlugin();
+
+await validate("https://schemas.hyperjump.io/main", { foo: 42 }, {
+  outputFormat: BASIC,
+  plugins: [evaluatedKeywordPlugin]
+});
+
+console.log(evaluatedKeywordPlugin.schemaLocations);
+// Set(4) {
+//   'https://schemas.hyperjump.io/main#/type',
+//   'https://schemas.hyperjump.io/main#/properties',
+//   'https://schemas.hyperjump.io/main#/properties/foo/type',
+//   'https://schemas.hyperjump.io/main#/required'
+// }
+
+// NOTE: #/properties/bar is not in the list because the instance doesn't include that property.
+```
+
 ### API
 
 These are available from any of the exports that refer to a version of JSON
@@ -212,11 +263,11 @@ Schema, such as `@hyperjump/json-schema/draft-2020-12`.
     Load a schema manually rather than fetching it from the filesystem or over
     the network. Any schema already registered with the same identifier will be
     replaced with no warning.
-* **validate**: (schemaURI: string, instance: any, outputFormat: OutputFormat = FLAG) => Promise\<OutputUnit>
+* **validate**: (schemaURI: string, instance: any, outputFormat: ValidationOptions | OutputFormat = FLAG) => Promise\<OutputUnit>
 
     Validate an instance against a schema. This function is curried to allow
     compiling the schema once and applying it to multiple instances.
-* **validate**: (schemaURI: string) => Promise\<(instance: any, outputFormat: OutputFormat = FLAG) => OutputUnit>
+* **validate**: (schemaURI: string) => Promise\<(instance: any, outputFormat: ValidationOptions | OutputFormat = FLAG) => OutputUnit>
 
     Compiling a schema to a validation function.
 * **FLAG**: "FLAG"
@@ -255,6 +306,16 @@ The following types are used in the above definitions
     Output is an experimental feature of the JSON Schema specification. There
     may be additional fields present in the OutputUnit, but only the `valid`
     property should be considered part of the Stable API.
+* **ValidationOptions**:
+
+    * outputFormat?: OutputFormat
+    * plugins?: EvaluationPlugin[]
+
+* **EvaluationPlugin**: object
+    * beforeSchema?(url: string, instance: JsonNode, context: Context): void
+    * beforeKeyword?(keywordNode: Node<any>, instance: JsonNode, context: Context, schemaContext: Context, keyword: Keyword): void
+    * afterKeyword?(keywordNode: Node<any>, instance: JsonNode, context: Context, valid: boolean, schemaContext: Context, keyword: Keyword): void
+    * afterSchema?(url: string, instance: JsonNode, context: Context, valid: boolean): void
 
 ## Bundling
 
@@ -546,12 +607,6 @@ These are available from the `@hyperjump/json-schema/experimental` export.
     * **ValidationContext**: object
       * ast: AST
       * plugins: EvaluationPlugins[]
-
-    * **EvaluationPlugin**: object
-      * beforeSchema(url: string, instance: JsonNode, context: Context): void
-      * beforeKeyword(keywordNode: Node<any>, instance: JsonNode, context: Context, schemaContext: Context, keyword: Keyword): void
-      * afterKeyword(keywordNode: Node<any>, instance: JsonNode, context: Context, valid: boolean, schemaContext: Context, keyword: Keyword): void
-      * afterSchema(url: string, instance: JsonNode, context: Context, valid: boolean): void
 * **defineVocabulary**: (id: string, keywords: { [keyword: string]: string }) => void
 
     Define a vocabulary that maps keyword name to keyword URIs defined using

annotations/index.d.ts

@@ -1,18 +1,18 @@
-import type { OutputFormat, OutputUnit } from "../lib/index.js";
+import type { OutputFormat, OutputUnit, ValidationOptions } from "../lib/index.js";
 import type { CompiledSchema } from "../lib/experimental.js";
 import type { JsonNode } from "../lib/json-node.js";
 import type { Json } from "@hyperjump/json-pointer";
 
 
 export const annotate: (
-  (schemaUrl: string, value: Json, outputFormat?: OutputFormat) => Promise<JsonNode>
+  (schemaUrl: string, value: Json, options?: OutputFormat | ValidationOptions) => Promise<JsonNode>
 ) & (
   (schemaUrl: string) => Promise<Annotator>
 );
 
-export type Annotator = (value: Json, outputFormat?: OutputFormat) => JsonNode;
+export type Annotator = (value: Json, options?: OutputFormat | ValidationOptions) => JsonNode;
 
-export const interpret: (compiledSchema: CompiledSchema, value: JsonNode, outputFormat?: OutputFormat) => JsonNode;
+export const interpret: (compiledSchema: CompiledSchema, value: JsonNode, options?: OutputFormat | ValidationOptions) => JsonNode;
 
 export class ValidationError extends Error {
   public output: OutputUnit;

annotations/index.js

@@ -1,50 +1,36 @@
-import { FLAG } from "../lib/index.js";
 import { ValidationError } from "./validation-error.js";
 import {
   getSchema,
   compile,
+  interpret as validate,
   BASIC,
-  DETAILED,
-  annotationsPlugin,
-  basicOutputPlugin,
-  detailedOutputPlugin
+  AnnotationsPlugin
 } from "../lib/experimental.js";
-import Validation from "../lib/keywords/validation.js";
 import * as Instance from "../lib/instance.js";
 
 
-export const annotate = async (schemaUri, json = undefined, outputFormat = undefined) => {
+export const annotate = async (schemaUri, json = undefined, options = undefined) => {
   const schema = await getSchema(schemaUri);
   const compiled = await compile(schema);
-  const interpretAst = (json, outputFormat) => interpret(compiled, Instance.fromJs(json), outputFormat);
+  const interpretAst = (json, options) => interpret(compiled, Instance.fromJs(json), options);
 
-  return json === undefined ? interpretAst : interpretAst(json, outputFormat);
+  return json === undefined ? interpretAst : interpretAst(json, options);
 };
 
-export const interpret = ({ ast, schemaUri }, instance, outputFormat = BASIC) => {
-  const context = { ast, plugins: [annotationsPlugin, ...ast.plugins] };
-
-  switch (outputFormat) {
-    case FLAG:
-      break;
-    case BASIC:
-      context.plugins.push(basicOutputPlugin);
-      break;
-    case DETAILED:
-      context.plugins.push(detailedOutputPlugin);
-      break;
-    default:
-      throw Error(`Unsupported output format '${outputFormat}'`);
-  }
+export const interpret = (compiledSchema, instance, options = BASIC) => {
+  const annotationsPlugin = new AnnotationsPlugin();
+  const plugins = options.plugins ?? [];
 
-  const valid = Validation.interpret(schemaUri, instance, context);
+  const output = validate(compiledSchema, instance, {
+    outputFormat: typeof options === "string" ? options : options.outputFormat ?? BASIC,
+    plugins: [annotationsPlugin, ...plugins]
+  });
 
-  if (!valid) {
-    const result = !valid && "errors" in context ? { valid, errors: context.errors } : { valid };
-    throw new ValidationError(result);
+  if (!output.valid) {
+    throw new ValidationError(output);
   }
 
-  for (const annotation of context.annotations) {
+  for (const annotation of annotationsPlugin.annotations) {
     const node = Instance.get(annotation.instanceLocation, instance);
     const keyword = annotation.keyword;
     if (!node.annotations[keyword]) {

bundle/generate-snapshots.js

@@ -1,6 +1,6 @@
 import { writeFile, mkdir, rm } from "node:fs/promises";
 import { isCompatible, md5, loadSchemas, testSuite, unloadSchemas } from "./test-utils.js";
-import { annotationsPlugin, compile, detailedOutputPlugin, getSchema, Validation } from "../lib/experimental.js";
+import { AnnotationsPlugin, compile, DetailedOutputPlugin, getSchema, Validation } from "../lib/experimental.js";
 import "../stable/index.js";
 import "../draft-2020-12/index.js";
 import "../draft-2019-09/index.js";
@@ -26,15 +26,17 @@ const snapshotGenerator = async (version, dialect) => {
 
       const schema = await getSchema(mainSchemaUri);
       const { ast, schemaUri } = await compile(schema);
+      const annotationsPlugin = new AnnotationsPlugin();
+      const detailedOutputPlugin = new DetailedOutputPlugin();
 
       const instance = Instance.fromJs(test.instance);
       const context = { ast, plugins: [detailedOutputPlugin, annotationsPlugin, ...ast.plugins] };
       const valid = Validation.interpret(schemaUri, instance, context);
 
       const expectedOutput = {
         valid,
-        errors: context.errors,
-        annotations: context.annotations
+        errors: detailedOutputPlugin.errors,
+        annotations: annotationsPlugin.annotations
       };
 
       unloadSchemas(testCase, mainSchemaUri);

bundle/test-suite.spec.ts

@@ -3,9 +3,9 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest";
 import { isCompatible, md5, loadSchemas, unloadSchemas, testSuite } from "./test-utils.js";
 import { registerSchema, unregisterSchema } from "../lib/index.js";
 import {
-  annotationsPlugin,
+  AnnotationsPlugin,
   compile,
-  detailedOutputPlugin,
+  DetailedOutputPlugin,
   getKeywordName,
   getSchema,
   Validation
@@ -65,6 +65,8 @@ const testRunner = (version: number, dialect: string) => {
                 const schema = await getSchema(mainSchemaUri);
                 const { ast, schemaUri } = await compile(schema);
 
+                const annotationsPlugin = new AnnotationsPlugin();
+                const detailedOutputPlugin = new DetailedOutputPlugin();
                 const instance = Instance.fromJs(test.instance);
                 const context = {
                   ast,
@@ -74,8 +76,8 @@ const testRunner = (version: number, dialect: string) => {
 
                 const output = {
                   valid,
-                  errors: context.errors,
-                  annotations: context.annotations
+                  errors: detailedOutputPlugin.errors,
+                  annotations: annotationsPlugin.annotations
                 };
 
                 const testId = md5(`${version}|${dialect}|${testCase.description}|${testIndex}`);

draft-2020-12/dynamicRef.js

@@ -32,10 +32,6 @@ const plugin = {
   },
   beforeKeyword(_url, _instance, context, schemaContext) {
     context.dynamicAnchors = schemaContext.dynamicAnchors;
-  },
-  afterKeyword() {
-  },
-  afterSchema() {
   }
 };
 

lib/core.js

@@ -11,19 +11,19 @@ import { InvalidSchemaError } from "./invalid-schema-error.js";
 import { getSchema, registerSchema, unregisterSchema as schemaUnregister } from "./schema.js";
 import { getKeywordName } from "./keywords.js";
 import Validation from "./keywords/validation.js";
-import { basicOutputPlugin } from "./evaluation-plugins/basic-output.js";
-import { detailedOutputPlugin } from "./evaluation-plugins/detailed-output.js";
+import { BasicOutputPlugin } from "./evaluation-plugins/basic-output.js";
+import { DetailedOutputPlugin } from "./evaluation-plugins/detailed-output.js";
 
 
 export const FLAG = "FLAG", BASIC = "BASIC", DETAILED = "DETAILED";
 setMetaSchemaOutputFormat(FLAG);
 
-export const validate = async (url, value = undefined, outputFormat = undefined) => {
+export const validate = async (url, value = undefined, options = undefined) => {
   const schema = await getSchema(url);
   const compiled = await compile(schema);
-  const interpretAst = (value, outputFormat) => interpret(compiled, Instance.fromJs(value), outputFormat);
+  const interpretAst = (value, options) => interpret(compiled, Instance.fromJs(value), options);
 
-  return value === undefined ? interpretAst : interpretAst(value, outputFormat);
+  return value === undefined ? interpretAst : interpretAst(value, options);
 };
 
 export const compile = async (schema) => {
@@ -32,24 +32,30 @@ export const compile = async (schema) => {
   return { ast, schemaUri };
 };
 
-export const interpret = curry(({ ast, schemaUri }, instance, outputFormat = FLAG) => {
-  const context = { ast, plugins: [...ast.plugins] };
+export const interpret = curry(({ ast, schemaUri }, instance, options = FLAG) => {
+  const outputFormat = typeof options === "string" ? options : options.outputFormat ?? FLAG;
+  const plugins = options.plugins ?? [];
 
+  const context = { ast, plugins: [...ast.plugins, ...plugins] };
+
+  let outputPlugin;
   switch (outputFormat) {
     case FLAG:
       break;
     case BASIC:
-      context.plugins.push(basicOutputPlugin);
+      outputPlugin = new BasicOutputPlugin();
+      context.plugins.push(outputPlugin);
       break;
     case DETAILED:
-      context.plugins.push(detailedOutputPlugin);
+      outputPlugin = new DetailedOutputPlugin();
+      context.plugins.push(outputPlugin);
       break;
     default:
       throw Error(`Unsupported output format '${outputFormat}'`);
   }
 
   const valid = Validation.interpret(schemaUri, instance, context);
-  return !valid && "errors" in context ? { valid, errors: context.errors } : { valid };
+  return !valid && outputPlugin ? { valid, errors: outputPlugin.errors } : { valid };
 });
 
 const metaValidators = {};

lib/evaluation-plugins/annotations.js

@@ -1,14 +1,16 @@
 import * as Instance from "../instance.js";
 
 
-export const annotationsPlugin = {
+export class AnnotationsPlugin {
   beforeSchema(_url, _instance, context) {
     context.annotations ??= [];
     context.schemaAnnotations = [];
-  },
+  }
+
   beforeKeyword(_node, _instance, context) {
     context.annotations = [];
-  },
+  }
+
   afterKeyword(node, instance, context, valid, schemaContext, keyword) {
     if (valid) {
       const [keywordId, schemaUri, keywordValue] = node;
@@ -23,10 +25,13 @@ export const annotationsPlugin = {
       }
       schemaContext.schemaAnnotations.push(...context.annotations);
     }
-  },
+  }
+
   afterSchema(_schemaNode, _instanceNode, context, valid) {
     if (valid) {
       context.annotations.push(...context.schemaAnnotations);
     }
+
+    this.annotations = context.annotations;
   }
-};
+}

lib/evaluation-plugins/basic-output.js

@@ -2,13 +2,15 @@ import { Validation } from "../experimental.js";
 import * as Instance from "../instance.js";
 
 
-export const basicOutputPlugin = {
+export class BasicOutputPlugin {
   beforeSchema(_url, _intance, context) {
     context.errors ??= [];
-  },
+  }
+
   beforeKeyword(_node, _instance, context) {
     context.errors = [];
-  },
+  }
+
   afterKeyword(node, instance, context, valid, schemaContext, keyword) {
     if (!valid) {
       if (!keyword.simpleApplicator) {
@@ -21,7 +23,8 @@ export const basicOutputPlugin = {
       }
       schemaContext.errors.push(...context.errors);
     }
-  },
+  }
+
   afterSchema(url, instance, context, valid) {
     if (typeof context.ast[url] === "boolean" && !valid) {
       context.errors.push({
@@ -30,5 +33,7 @@ export const basicOutputPlugin = {
         instanceLocation: Instance.uri(instance)
       });
     }
+
+    this.errors = context.errors;
   }
-};
+}