Source code

001/*
002 * Portions of this software was developed by employees of the National Institute
003 * of Standards and Technology (NIST), an agency of the Federal Government and is
004 * being made available as a public service. Pursuant to title 17 United States
005 * Code Section 105, works of NIST employees are not subject to copyright
006 * protection in the United States. This software may be subject to foreign
007 * copyright. Permission in the United States and in foreign countries, to the
008 * extent that NIST may hold copyright, to use, copy, modify, create derivative
009 * works, and distribute this software and its documentation without fee is hereby
010 * granted on a non-exclusive basis, provided that this notice and disclaimer
011 * of warranty appears in all copies.
012 *
013 * THE SOFTWARE IS PROVIDED 'AS IS' WITHOUT ANY WARRANTY OF ANY KIND, EITHER
014 * EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTY
015 * THAT THE SOFTWARE WILL CONFORM TO SPECIFICATIONS, ANY IMPLIED WARRANTIES OF
016 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND FREEDOM FROM
017 * INFRINGEMENT, AND ANY WARRANTY THAT THE DOCUMENTATION WILL CONFORM TO THE
018 * SOFTWARE, OR ANY WARRANTY THAT THE SOFTWARE WILL BE ERROR FREE.  IN NO EVENT
019 * SHALL NIST BE LIABLE FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO, DIRECT,
020 * INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES, ARISING OUT OF, RESULTING FROM,
021 * OR IN ANY WAY CONNECTED WITH THIS SOFTWARE, WHETHER OR NOT BASED UPON WARRANTY,
022 * CONTRACT, TORT, OR OTHERWISE, WHETHER OR NOT INJURY WAS SUSTAINED BY PERSONS OR
023 * PROPERTY OR OTHERWISE, AND WHETHER OR NOT LOSS WAS SUSTAINED FROM, OR AROSE OUT
024 * OF THE RESULTS OF, OR USE OF, THE SOFTWARE OR SERVICES PROVIDED HEREUNDER.
025 */
026
027package gov.nist.secauto.metaschema.model.testing;
028
029import static org.junit.jupiter.api.Assertions.assertEquals;
030
031import gov.nist.secauto.metaschema.core.model.IModule;
032import gov.nist.secauto.metaschema.core.model.MetaschemaException;
033import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
034import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
035import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
036import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
037import gov.nist.secauto.metaschema.core.model.xml.ModuleLoader;
038import gov.nist.secauto.metaschema.core.util.ObjectUtils;
039import gov.nist.secauto.metaschema.databind.DefaultBindingContext;
040import gov.nist.secauto.metaschema.databind.IBindingContext;
041import gov.nist.secauto.metaschema.databind.io.Format;
042import gov.nist.secauto.metaschema.databind.io.ISerializer;
043import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
044import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
045import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
046import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
047import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
048import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
049
050import org.apache.logging.log4j.LogBuilder;
051import org.apache.logging.log4j.LogManager;
052import org.apache.logging.log4j.Logger;
053import org.apache.xmlbeans.XmlException;
054import org.apache.xmlbeans.XmlOptions;
055import org.junit.jupiter.api.DynamicContainer;
056import org.junit.jupiter.api.DynamicNode;
057import org.junit.jupiter.api.DynamicTest;
058import org.junit.platform.commons.JUnitException;
059
060import java.io.IOException;
061import java.io.Writer;
062import java.net.URI;
063import java.net.URISyntaxException;
064import java.net.URL;
065import java.nio.charset.StandardCharsets;
066import java.nio.file.FileVisitResult;
067import java.nio.file.Files;
068import java.nio.file.OpenOption;
069import java.nio.file.Path;
070import java.nio.file.SimpleFileVisitor;
071import java.nio.file.StandardOpenOption;
072import java.nio.file.attribute.BasicFileAttributes;
073import java.util.concurrent.ExecutionException;
074import java.util.concurrent.ExecutorService;
075import java.util.concurrent.Executors;
076import java.util.concurrent.Future;
077import java.util.function.BiFunction;
078import java.util.function.Function;
079import java.util.function.Supplier;
080import java.util.stream.Stream;
081
082import edu.umd.cs.findbugs.annotations.NonNull;
083import edu.umd.cs.findbugs.annotations.Nullable;
084
085public abstract class AbstractTestSuite {
086  private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
087  private static final ModuleLoader LOADER = new ModuleLoader();
088
089  private static final boolean DELETE_RESULTS_ON_EXIT = false;
090
091  static {
092    LOADER.allowEntityResolution();
093  }
094
095  @NonNull
096  protected abstract Format getRequiredContentFormat();
097
098  @NonNull
099  protected abstract URI getTestSuiteURI();
100
101  @NonNull
102  protected abstract Path getGenerationPath();
103
104  @NonNull
105  protected abstract BiFunction<IModule, Writer, Void> getGeneratorSupplier();
106
107  @Nullable
108  protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
109
110  @NonNull
111  protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
112
113  protected Stream<DynamicNode> testFactory() {
114    try {
115      return generateTests();
116    } catch (XmlException | IOException ex) {
117      throw new JUnitException("Unable to generate tests", ex);
118    }
119  }
120
121  private Stream<DynamicNode> generateTests() throws XmlException, IOException {
122    XmlOptions options = new XmlOptions();
123    options.setBaseURI(null);
124    options.setLoadLineNumbers();
125
126    Path generationPath = getGenerationPath();
127    if (Files.exists(generationPath)) {
128      if (!Files.isDirectory(generationPath)) {
129        throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
130      }
131    } else {
132      Files.createDirectories(generationPath);
133    }
134
135    URI testSuiteUri = getTestSuiteURI();
136    URL testSuiteUrl = testSuiteUri.toURL();
137    TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
138    return directive.getTestSuite().getTestCollectionList().stream()
139        .flatMap(
140            collection -> Stream.of(generateCollection(ObjectUtils.notNull(collection), testSuiteUri, generationPath)));
141  }
142
143  protected void deleteCollectionOnExit(Path path) {
144    if (path != null) {
145      Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
146          new Runnable() {
147            @Override
148            public void run() {
149              try {
150                Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
151                  @Override
152                  public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
153                    Files.delete(file);
154                    return FileVisitResult.CONTINUE;
155                  }
156
157                  @Override
158                  public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
159                    if (ex == null) {
160                      Files.delete(dir);
161                      return FileVisitResult.CONTINUE;
162                    }
163                    // directory iteration failed for some reason
164                    throw ex;
165                  }
166                });
167              } catch (IOException ex) {
168                throw new IllegalStateException("Failed to delete collection: " + path, ex);
169              }
170            }
171          }));
172    }
173  }
174
175  private DynamicContainer generateCollection(@NonNull TestCollection collection, @NonNull URI testSuiteUri,
176      @NonNull Path generationPath) {
177    URI collectionUri = testSuiteUri.resolve(collection.getLocation());
178    assert collectionUri != null;
179
180    LOGGER.atInfo().log("Collection: " + collectionUri);
181    Path collectionGenerationPath;
182    try {
183      collectionGenerationPath = ObjectUtils.notNull(Files.createTempDirectory(generationPath, "collection-"));
184      assert collectionGenerationPath != null;
185      if (DELETE_RESULTS_ON_EXIT) {
186        deleteCollectionOnExit(collectionGenerationPath);
187      }
188    } catch (IOException ex) {
189      throw new JUnitException("Unable to create collection temp directory", ex);
190    }
191
192    return DynamicContainer.dynamicContainer(
193        collection.getName(),
194        testSuiteUri,
195        collection.getTestScenarioList().stream()
196            .flatMap(scenario -> {
197              assert scenario != null;
198              return Stream.of(generateScenario(scenario, collectionUri, collectionGenerationPath));
199            })
200            .sequential());
201  }
202
203  protected void produceSchema(@NonNull IModule module, @NonNull Path schemaPath) throws IOException {
204    produceSchema(module, schemaPath, getGeneratorSupplier());
205  }
206
207  protected void produceSchema(@NonNull IModule module, @NonNull Path schemaPath,
208      @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
209    Path parentDir = schemaPath.getParent();
210    if (parentDir != null && !Files.exists(parentDir)) {
211      Files.createDirectories(parentDir);
212    }
213
214    try (Writer writer = Files.newBufferedWriter(
215        schemaPath,
216        StandardCharsets.UTF_8,
217        getWriteOpenOptions())) {
218      schemaProducer.apply(module, writer);
219    }
220    LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
221  }
222
223  protected OpenOption[] getWriteOpenOptions() {
224    return new OpenOption[] {
225        StandardOpenOption.CREATE,
226        StandardOpenOption.WRITE,
227        StandardOpenOption.TRUNCATE_EXISTING
228    };
229  }
230
231  private DynamicContainer generateScenario(@NonNull TestScenario scenario, @NonNull URI collectionUri,
232      @NonNull Path collectionGenerationPath) {
233    Path scenarioGenerationPath;
234    try {
235      scenarioGenerationPath = Files.createTempDirectory(collectionGenerationPath, "scenario-");
236    } catch (IOException ex) {
237      throw new JUnitException("Unable to create scenario temp directory", ex);
238    }
239
240    try {
241      // create the directories the schema will be stored in
242      Files.createDirectories(scenarioGenerationPath);
243    } catch (IOException ex) {
244      throw new JUnitException("Unable to create test directories for path: " + scenarioGenerationPath, ex);
245    }
246
247    GenerateSchema generateSchema = scenario.getGenerateSchema();
248    MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
249    URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
250
251    ExecutorService executor = Executors.newSingleThreadExecutor(); // NOPMD - intentional use of threads
252    Future<IModule> loadModuleFuture = executor.submit(() -> {
253      IModule module;
254      try {
255        module = LOADER.load(ObjectUtils.notNull(metaschemaUri.toURL()));
256      } catch (IOException | MetaschemaException ex) {
257        throw new JUnitException("Unable to generate schema for Module: " + metaschemaUri, ex);
258      }
259      return module;
260    });
261
262    Future<Path> generateSchemaFuture = executor.submit(() -> {
263      Path schemaPath;
264      // determine what file to use for the schema
265      try {
266        schemaPath = Files.createTempFile(scenarioGenerationPath, "", "-schema");
267      } catch (IOException ex) {
268        throw new JUnitException("Unable to create schema temp file", ex);
269      }
270      IModule module = loadModuleFuture.get();
271      produceSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath));
272      return schemaPath;
273    });
274
275    Future<IBindingContext> dynamicBindingContextFuture = executor.submit(() -> {
276      IModule module = loadModuleFuture.get();
277      IBindingContext context;
278      try {
279        context = new DefaultBindingContext();
280        context.registerModule(
281            ObjectUtils.notNull(module),
282            ObjectUtils.notNull(scenarioGenerationPath));
283      } catch (Exception ex) { // NOPMD - intentional
284        throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
285      }
286      return context;
287    });
288    assert dynamicBindingContextFuture != null;
289
290    Future<IContentValidator> contentValidatorFuture = executor.submit(() -> {
291      Path schemaPath = generateSchemaFuture.get();
292      return getContentValidatorSupplier().apply(schemaPath);
293    });
294    assert contentValidatorFuture != null;
295
296    // build a test container for the generate and validate steps
297    DynamicTest validateSchema = DynamicTest.dynamicTest(
298        "Validate Schema",
299        () -> {
300          Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
301          if (supplier != null) {
302            Path schemaPath;
303            try {
304              schemaPath = ObjectUtils.requireNonNull(generateSchemaFuture.get());
305            } catch (ExecutionException ex) {
306              throw new JUnitException( // NOPMD - cause is relevant, exception is not
307                  "failed to generate schema", ex.getCause());
308            }
309            validate(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
310          }
311        });
312
313    Stream<? extends DynamicNode> contentTests;
314    {
315      contentTests = scenario.getValidationCaseList().stream()
316          .flatMap(contentCase -> {
317            assert contentCase != null;
318            DynamicTest test
319                = generateValidationCase(contentCase, dynamicBindingContextFuture, contentValidatorFuture,
320                    collectionUri, ObjectUtils.notNull(scenarioGenerationPath));
321            return test == null ? Stream.empty() : Stream.of(test);
322          }).sequential();
323    }
324
325    return DynamicContainer.dynamicContainer(
326        scenario.getName(),
327        metaschemaUri,
328        Stream.concat(Stream.of(validateSchema), contentTests).sequential());
329  }
330
331  @SuppressWarnings("unchecked")
332  protected Path convertContent(URI contentUri, @NonNull Path generationPath, @NonNull IBindingContext context)
333      throws IOException {
334    Object object;
335    try {
336      object = context.newBoundLoader().load(ObjectUtils.notNull(contentUri.toURL()));
337    } catch (URISyntaxException ex) {
338      throw new IOException(ex);
339    }
340
341    if (!Files.exists(generationPath)) {
342      Files.createDirectories(generationPath);
343    }
344
345    Path convertedContetPath;
346    try {
347      convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
348    } catch (IOException ex) {
349      throw new JUnitException("Unable to create schema temp file", ex);
350    }
351
352    @SuppressWarnings("rawtypes") ISerializer serializer
353        = context.newSerializer(getRequiredContentFormat(), object.getClass());
354    serializer.serialize(object, convertedContetPath, getWriteOpenOptions());
355
356    return convertedContetPath;
357  }
358
359  private DynamicTest generateValidationCase(
360      @NonNull ContentCaseType contentCase,
361      @NonNull Future<IBindingContext> contextFuture,
362      @NonNull Future<IContentValidator> contentValidatorFuture,
363      @NonNull URI collectionUri,
364      @NonNull Path generationPath) {
365
366    URI contentUri = collectionUri.resolve(contentCase.getLocation());
367
368    Format format = contentCase.getSourceFormat();
369    DynamicTest retval = null;
370    if (getRequiredContentFormat().equals(format)) {
371      retval = DynamicTest.dynamicTest(
372          String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
373              contentCase.getLocation()),
374          contentUri,
375          () -> {
376            IContentValidator contentValidator;
377            try {
378              contentValidator = contentValidatorFuture.get();
379            } catch (ExecutionException ex) {
380              throw new JUnitException( // NOPMD - cause is relevant, exception is not
381                  "failed to produce the content validator", ex.getCause());
382            }
383
384            assertEquals(
385                contentCase.getValidationResult(),
386                validate(
387                    ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
388                "validation did not match expectation");
389          });
390    } else if (contentCase.getValidationResult()) {
391      retval = DynamicTest.dynamicTest(
392          String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
393              contentCase.getLocation()),
394          contentUri,
395          () -> {
396            IBindingContext context;
397            try {
398              context = contextFuture.get();
399            } catch (ExecutionException ex) {
400              throw new JUnitException( // NOPMD - cause is relevant, exception is not
401                  "failed to produce the content validator", ex.getCause());
402            }
403            assert context != null;
404
405            Path convertedContetPath;
406            try {
407              convertedContetPath = convertContent(contentUri, generationPath, context);
408            } catch (Exception ex) { // NOPMD - intentional
409              throw new JUnitException("failed to convert content: " + contentUri, ex);
410            }
411
412            IContentValidator contentValidator;
413            try {
414              contentValidator = contentValidatorFuture.get();
415            } catch (ExecutionException ex) {
416              throw new JUnitException( // NOPMD - cause is relevant, exception is not
417                  "failed to produce the content validator",
418                  ex.getCause());
419            }
420            assertEquals(contentCase.getValidationResult(),
421                validate(
422                    ObjectUtils.notNull(contentValidator),
423                    ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
424                String.format("validation of '%s' did not match expectation", convertedContetPath));
425          });
426    }
427    return retval;
428  }
429
430  private static boolean validate(@NonNull IContentValidator validator, @NonNull URL target) throws IOException {
431    IValidationResult schemaValidationResult;
432    try {
433      schemaValidationResult = validator.validate(target);
434    } catch (URISyntaxException ex) {
435      throw new IOException(ex);
436    }
437    return processValidationResult(schemaValidationResult);
438  }
439
440  protected static boolean validate(@NonNull IContentValidator validator, @NonNull Path target) throws IOException {
441    IValidationResult schemaValidationResult = validator.validate(target);
442    if (!schemaValidationResult.isPassing()) {
443      LOGGER.atError().log("Schema validation failed for: {}", target);
444    }
445    return processValidationResult(schemaValidationResult);
446  }
447
448  private static boolean processValidationResult(IValidationResult schemaValidationResult) {
449    for (IValidationFinding finding : schemaValidationResult.getFindings()) {
450      logFinding(ObjectUtils.notNull(finding));
451    }
452    return schemaValidationResult.isPassing();
453  }
454
455  private static void logFinding(@NonNull IValidationFinding finding) {
456    LogBuilder logBuilder;
457    switch (finding.getSeverity()) {
458    case CRITICAL:
459      logBuilder = LOGGER.atFatal();
460      break;
461    case ERROR:
462      logBuilder = LOGGER.atError();
463      break;
464    case WARNING:
465      logBuilder = LOGGER.atWarn();
466      break;
467    case INFORMATIONAL:
468      logBuilder = LOGGER.atInfo();
469      break;
470    default:
471      throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
472    }
473
474    // if (finding.getCause() != null) {
475    // logBuilder.withThrowable(finding.getCause());
476    // }
477
478    if (finding instanceof JsonValidationFinding) {
479      JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
480      logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
481    } else {
482      logBuilder.log("{}", finding.getMessage());
483    }
484  }
485}