Class: EarlReport

Inherits:
Object
  • Object
show all
Defined in:
lib/earl_report.rb

Overview

EARL reporting class. Instantiate a new class using one or more input graphs

Defined Under Namespace

Modules: VERSION Classes: MF

Constant Summary collapse

MANIFEST_QUERY =

Return information about each test. Tests all have an mf:action property. The Manifest lists all actions in list from mf:entries

%(
  PREFIX mf: <http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#>
  PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>

  SELECT ?uri ?testAction ?manUri
  WHERE {
    ?uri mf:action ?testAction .
    OPTIONAL {
      ?manUri a mf:Manifest; mf:entries ?lh .
      ?lh rdf:first ?uri .
    }
  }
).freeze
TEST_SUBJECT_QUERY = 
%(
  PREFIX doap: <http://usefulinc.com/ns/doap#>
  PREFIX foaf: <http://xmlns.com/foaf/0.1/>

  SELECT DISTINCT ?uri ?name ?doapDesc ?release ?revision ?homepage ?language ?developer ?devName ?devType ?devHomepage
  WHERE {
    ?uri a doap:Project; doap:name ?name; doap:developer ?developer .
    OPTIONAL { ?uri doap:homepage ?homepage . }
    OPTIONAL { ?uri doap:description ?doapDesc . }
    OPTIONAL { ?uri doap:programming-language ?language . }
    OPTIONAL { ?uri doap:release ?release . }
    OPTIONAL { ?release doap:revision ?revision .}
    OPTIONAL { ?developer a ?devType .}
    OPTIONAL { ?developer foaf:name ?devName .}
    OPTIONAL { ?developer foaf:homepage ?devHomepage .}
  }
  ORDER BY ?name
).freeze
DOAP_QUERY = 
%(
  PREFIX earl: <http://www.w3.org/ns/earl#>
  PREFIX doap: <http://usefulinc.com/ns/doap#>
  
  SELECT DISTINCT ?subject ?name
  WHERE {
    [ a earl:Assertion; earl:subject ?subject ] .
    OPTIONAL {
      ?subject a doap:Project; doap:name ?name
    }
  }
).freeze
ASSERTION_QUERY = 
%(
  PREFIX earl: <http://www.w3.org/ns/earl#>
  
  SELECT ?test ?subject ?by ?mode ?outcome ?info
  WHERE {
    ?a a earl:Assertion;
      earl:assertedBy ?by;
      earl:result ?result;
      earl:subject ?subject;
      earl:test ?test .
    ?result earl:outcome ?outcome .
    OPTIONAL {?a earl:mode ?mode .}
    OPTIONAL {?result earl:info ?info .}
  }
  ORDER BY ?subject
).freeze
TEST_FRAME = 
{
  "@context" => {
    "@version" =>     1.1,
    "@vocab" =>       "http://www.w3.org/ns/earl#",
    "foaf:homepage"=> {"@type" => "@id"},
    "dc" =>           "http://purl.org/dc/terms/",
    "doap" =>         "http://usefulinc.com/ns/doap#",
    "earl" =>         "http://www.w3.org/ns/earl#",
    "mf" =>           "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#",
    "foaf" =>         "http://xmlns.com/foaf/0.1/",
    "rdfs" =>         "http://www.w3.org/2000/01/rdf-schema#",
    "assertedBy" =>   {"@type" => "@id"},
    "assertions" =>   {"@id" => "mf:report", "@type" => "@id", "@container" => "@set"},
    "bibRef" =>       {"@id" => "dc:bibliographicCitation"},
    "created" =>      {"@id" => "doap:created", "@type" => "xsd:date"},
    "description" =>  {"@id" => "rdfs:comment", "@language" => "en"},
    "developer" =>    {"@id" => "doap:developer", "@type" => "@id", "@container" => "@set"},
    "doapDesc" =>     {"@id" => "doap:description", "@language" => "en"},
    "entries" =>      {
      "@id" => "mf:entries", "@type" => "@id", "@container" => "@list",
      "@context" => {
        "assertions" =>   {
          "@reverse" => "earl:test", "@type" => "@id", "@container" => "@set"
        },
      }
    },
    "generatedBy" =>  {"@type" => "@id"},
    "homepage" =>     {"@id" => "doap:homepage", "@type" => "@id"},
    "language" =>     {"@id" => "doap:programming-language"},
    "license" =>      {"@id" => "doap:license", "@type" => "@id"},
    "mode" =>         {"@type" => "@id"},
    "name" =>         {"@id" => "doap:name"},
    "outcome" =>      {"@type" => "@id"},
    "release" =>      {"@id" => "doap:release", "@type" => "@id"},
    "revision" =>     {"@id" => "doap:revision"},
    "shortdesc" =>    {"@id" => "doap:shortdesc", "@language" => "en"},
    "subject" =>      {"@type" => "@id"},
    "test" =>         {"@type" => "@id"},
    "testAction" =>   {"@id" => "mf:action", "@type" => "@id"},
    "testResult" =>   {"@id" => "mf:result", "@type" => "@id"},
    "title" =>        {"@id" => "mf:name"},
    "testSubjects" => {"@type" => "@id", "@container" => "@set"},
    "xsd" =>          {"@id" => "http://www.w3.org/2001/XMLSchema#"}
  },
  "@requireAll" => true,
  "@embed" => "@always",
  "assertions" => {},
  "bibRef" => {},
  "generatedBy" => {
    "@embed" => "@always",
    "developer" => {"@embed" => "@always"},
    "release" => {"@embed" => "@always"}
  },
  "testSubjects" => {
    "@embed" => "@always",
    "@requireAll" => false,
    "@type" => "earl:TestSubject",
    "developer" => {"@embed" => "@always"},
    "release" => {"@embed" => "@always"},
    "homepage" => {"@embed" => "@never"}
  },
  "entries" => [{
    "@embed" => "@always",
    "@type" => "mf:Manifest",
    "entries" => [{
      "@embed" => "@always",
      "@type" => "earl:TestCase",
      "assertions" => {
        "@embed" => "@always",
        "@type" => "earl:Assertion",
        "assertedBy" => {"@embed" => "@never"},
        "result" => {
          "@embed" => "@always",
          "@type" => "earl:TestResult"
        },
        "subject" => {"@embed" => "@never"}
      }
    }]
  }]
}.freeze
TURTLE_PREFIXES = 
%(@prefix dc:   <http://purl.org/dc/terms/> .
@prefix doap: <http://usefulinc.com/ns/doap#> .
@prefix earl: <http://www.w3.org/ns/earl#> .
@prefix mf:   <http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#> .
@prefix xsd:  <http://www.w3.org/2001/XMLSchema#> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
@prefix rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
).gsub(/^  /, '')
TURTLE_SOFTWARE = 
%(
# Report Generation Software
<https://rubygems.org/gems/earl-report> a earl:Software, doap:Project;
   doap:name "earl-report";
   doap:shortdesc "Earl Report summary generator"@en;
   doap:description "EarlReport generates HTML+RDFa rollups of multiple EARL reports"@en;
   doap:homepage <https://github.com/gkellogg/earl-report>;
   doap:programming-language "Ruby";
   doap:license <http://unlicense.org>;
   doap:release <https://github.com/gkellogg/earl-report/tree/#{VERSION}>;
   doap:developer <https://greggkellogg.net/foaf#me> .

<https://github.com/gkellogg/earl-report/tree/#{VERSION}> a doap:Version;
  doap:name "earl-report-#{VERSION}";
  doap:created "#{File.mtime(File.expand_path('../../VERSION', __FILE__)).strftime('%Y-%m-%d')}"^^xsd:date;
  doap:revision "#{VERSION}" .
).gsub(/^  /, '')

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*files, base: nil, bibRef: 'Unknown reference', json: false, manifest: nil, name: 'Unknown', query: MANIFEST_QUERY, strict: false, verbose: false, **options) ⇒ EarlReport

Load test assertions and look for referenced software and developer information

Parameters:

  • files (Array<String>) —

    Assertions

  • base (String) (defaults to: nil) —

    (nil) Base IRI for loading Manifest

  • bibRef (String) (defaults to: 'Unknown reference') —

    (‘Unknown reference’) ReSpec bibliography reference for specification being tested

  • json (Boolean) (defaults to: false) —

    (false) File is in the JSON format of a report.

  • manifest (String, Array<String>) (defaults to: nil) —

    (nil) Test manifest(s)

  • name (String) (defaults to: 'Unknown') —

    (‘Unknown’) Name of specification

  • query (String) (defaults to: MANIFEST_QUERY) —

    (MANIFEST_QUERY) Query, or file containing query for extracting information from Test manifests

  • strict (Boolean) (defaults to: false) —

    (false) Abort on any warning

  • verbose (Boolean) (defaults to: false) —

    (false)



217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
 
# File 'lib/earl_report.rb', line 217

def initialize(*files,
               base: nil,
               bibRef: 'Unknown reference',
               json: false,
               manifest: nil,
               name: 'Unknown',
               query: MANIFEST_QUERY,
               strict: false,
               verbose: false,
               **options)
  @verbose = verbose
  raise "Test Manifests must be specified with :manifest option" unless manifest || json
  raise "Require at least one input file" if files.empty?
  @files = files
  @prefixes = {}
  @warnings = 0

  # If provided json, it is used for generating all other output forms
  if json
    @json_hash = ::JSON.parse(File.read(files.first))
    # Add a base_uri so relative subjects aren't dropped
    JSON::LD::Reader.open(files.first, base_uri: "http://example.org/report") do |r|
      @graph = RDF::Graph.new
      r.each_statement do |statement|
        # restore relative subject
        statement.subject = RDF::URI("") if statement.subject == "http://example.org/report"
        @graph << statement
      end
    end
    return
  end

  # Load manifests, possibly with base URI
  status "read #{manifest.inspect}"
  man_opts = {}
  man_opts[:base_uri] = RDF::URI(base) if base
  @graph = RDF::Graph.new
  Array(manifest).each do |man|
    g = RDF::Graph.load(man, unique_bnodes: true, **man_opts)
    status "  loaded #{g.count} triples from #{man}"
    graph << g
  end

  # Hash test cases by URI
  tests = SPARQL.execute(query, graph)
    .to_a
    .inject({}) {|memo, soln| memo[soln[:uri]] = soln; memo}

  if tests.empty?
    raise "no tests found querying manifest.\n" +
          "Results are found using the following query, this can be overridden using the --query option:\n" +
          "#{query}"
  end

  # Manifests in graph
  man_uris = tests.values.map {|v| v[:manUri]}.uniq.compact
  test_resources = tests.values.map {|v| v[:uri]}.uniq.compact
  subjects = {}

  # Initialize test assertions with an entry for each test subject
  test_assertion_lists = {}
  test_assertion_lists = tests.keys.inject({}) do |memo, test|
    memo.merge(test => [])
  end

  assertion_stats = {}

  # Read test assertion files into assertion graph
  files.flatten.each do |file|
    status "read #{file}"
    file_graph = RDF::Graph.load(file)
    if file_graph.first_object(predicate: RDF::URI('http://www.w3.org/ns/earl#testSubjects'))
      warn "   skip #{file}, which seems to be a previous rollup earl report"
      @files -= [file]
    else
      status "  loaded #{file_graph.count} triples"

      # Find or load DOAP descriptions for all subjects
      SPARQL.execute(DOAP_QUERY, file_graph).each do |solution|
        subject = solution[:subject]

        # Load DOAP definitions
        unless solution[:name] # not loaded
          status "  read doap description for #{subject}"
          begin
            doap_graph = RDF::Graph.load(subject)
            status "    loaded #{doap_graph.count} triples"
            file_graph << doap_graph.to_a
          rescue
            warn "\nfailed to load DOAP from #{subject}: #{$!}"
          end
        end
      end

      # Sanity check loaded graph, look for test subject
      solutions = SPARQL.execute(TEST_SUBJECT_QUERY, file_graph)
      if solutions.empty?
        warn "\nTest subject info not found for #{file}, expect DOAP description of project solving the following query:\n" +
          TEST_SUBJECT_QUERY
        next
      end

      # Load developers referenced from Test Subjects
      if !solutions.all? {|s| s[:developer]}
        warn "\nNo developer identified for #{solutions.first[:uri]}"
      elsif !solutions.all? {|s| s[:devName]}
        solutions.reject {|s| s[:devName]}.each do |no_foaf|
          status "  read description for developer #{no_foaf[:developer].inspect}"
          begin
            foaf_graph = RDF::Graph.load(no_foaf[:developer])
            status "    loaded #{foaf_graph.count} triples"
            file_graph << foaf_graph.to_a
          rescue
            warn "\nfailed to load FOAF from #{no_foaf[:developer]}: #{$!}"
          end
        end

        # Reload solutions
        solutions = SPARQL.execute(TEST_SUBJECT_QUERY, file_graph)
      end

      release = nil
      solutions.each do |solution|
        # Kepp track of subjects
        subjects[solution[:uri]] = RDF::URI(file)

        # Add TestSubject information to main graph
        doapName = solution[:name].to_s if solution[:name]
        language = solution[:language].to_s if solution[:language]
        doapDesc = solution[:doapDesc] if solution[:doapDesc]
        doapDesc.language ||= :en if doapDesc
        devName = solution[:devName].to_s if solution[:devName]
        graph << RDF::Statement(solution[:uri], RDF.type, RDF::Vocab::DOAP.Project)
        graph << RDF::Statement(solution[:uri], RDF.type, RDF::Vocab::EARL.TestSubject)
        graph << RDF::Statement(solution[:uri], RDF.type, RDF::Vocab::EARL.Software)
        graph << RDF::Statement(solution[:uri], RDF::Vocab::DOAP.name, doapName)
        graph << RDF::Statement(solution[:uri], RDF::Vocab::DOAP.developer, solution[:developer])
        graph << RDF::Statement(solution[:uri], RDF::Vocab::DOAP.homepage, solution[:homepage]) if solution[:homepage]
        graph << RDF::Statement(solution[:uri], RDF::Vocab::DOAP.description, doapDesc) if doapDesc
        graph << RDF::Statement(solution[:uri], RDF::Vocab::DOAP[:"programming-language"], language) if solution[:language]
        graph << RDF::Statement(solution[:developer], RDF.type, solution[:devType]) if solution[:devType]
        graph << RDF::Statement(solution[:developer], RDF::Vocab::FOAF.name, devName) if devName
        graph << RDF::Statement(solution[:developer], RDF::Vocab::FOAF.homepage, solution[:devHomepage]) if solution[:devHomepage]

        # Make sure BNode identifiers don't leak
        release ||= if !solution[:release] || solution[:release].node?
          RDF::Node.new
        else
          solution[:release]
        end
        graph << RDF::Statement(solution[:uri], RDF::Vocab::DOAP.release, release)
        graph << RDF::Statement(release, RDF::Vocab::DOAP.revision, (solution[:revision] || "unknown"))
      end

      # Make sure that each assertion matches a test and add reference from test to assertion
      found_solutions = false
      subject = nil

      status "  query assertions"
      SPARQL.execute(ASSERTION_QUERY, file_graph).each do |solution|
        subject = solution[:subject]
        unless tests[solution[:test]]
          assertion_stats["Skipped"] = assertion_stats["Skipped"].to_i + 1
          warn "Skipping result for #{solution[:test]} for #{subject}, which is not defined in manifests"
          next
        end
        unless subjects[subject]
          assertion_stats["Missing Subject"] = assertion_stats["Missing Subject"].to_i + 1
          warn "No test result subject found for #{subject}: in #{subjects.keys.join(', ')}"
          next
        end
        found_solutions ||= true
        assertion_stats["Found"] = assertion_stats["Found"].to_i + 1

        # Add this solution at the appropriate index within that list
        ndx = subjects.keys.find_index(subject)
        ary = test_assertion_lists[solution[:test]]

        ary[ndx] = a = RDF::Node.new
        graph << RDF::Statement(a, RDF.type, RDF::Vocab::EARL.Assertion)
        graph << RDF::Statement(a, RDF::Vocab::EARL.subject, subject)
        graph << RDF::Statement(a, RDF::Vocab::EARL.test, solution[:test])
        graph << RDF::Statement(a, RDF::Vocab::EARL.assertedBy, solution[:by])
        graph << RDF::Statement(a, RDF::Vocab::EARL.mode, solution[:mode]) if solution[:mode]
        r = RDF::Node.new
        graph << RDF::Statement(a, RDF::Vocab::EARL.result, r)
        graph << RDF::Statement(r, RDF.type, RDF::Vocab::EARL.TestResult)
        graph << RDF::Statement(r, RDF::Vocab::EARL.outcome, solution[:outcome])
        graph << RDF::Statement(r, RDF::Vocab::EARL.info, solution[:info]) if solution[:info]
      end

      # See if subject did not report results, which may indicate a formatting error in the EARL source
      warn "No results found for #{subject} using #{ASSERTION_QUERY}" unless found_solutions
    end
  end

  # Add ordered assertions for each test
  test_assertion_lists.each do |test, ary|
    ary[subjects.length - 1] ||= nil # extend for all subjects
    # Fill any missing entries with an untested outcome
    ary.each_with_index do |a, ndx|
      unless a
        assertion_stats["Untested"] = assertion_stats["Untested"].to_i + 1
        ary[ndx] = a = RDF::Node.new
        graph << RDF::Statement(a, RDF.type, RDF::Vocab::EARL.Assertion)
        graph << RDF::Statement(a, RDF::Vocab::EARL.subject, subjects.keys[ndx])
        graph << RDF::Statement(a, RDF::Vocab::EARL.test, test)
        r = RDF::Node.new
        graph << RDF::Statement(a, RDF::Vocab::EARL.result, r)
        graph << RDF::Statement(r, RDF.type, RDF::Vocab::EARL.TestResult)
        graph << RDF::Statement(r, RDF::Vocab::EARL.outcome, RDF::Vocab::EARL.untested)
      end
    end
  end

  assertion_stats.each {|stat, count| status("Assertions #{stat}: #{count}")}

  # Add report wrapper to graph
  ttl = TURTLE_PREFIXES + %(
  <> a earl:Software, doap:Project;
  doap:name #{quoted(name)};
  dc:bibliographicCitation "#{bibRef}";
  earl:generatedBy <https://rubygems.org/gems/earl-report>;
  mf:report #{subjects.values.map {|f| f.to_ntriples}.join(",\n          ")};
  earl:testSubjects #{subjects.keys.map {|f| f.to_ntriples}.join(",\n          ")};
  mf:entries (#{man_uris.map {|f| f.to_ntriples}.join("\n          ")}) .
  ).gsub(/^    /, '') +
    TURTLE_SOFTWARE
  RDF::Turtle::Reader.new(ttl) {|r| graph << r}

  # Each manifest is an earl:Report
  man_uris.each do |u|
    graph << RDF::Statement.new(u, RDF.type, RDF::Vocab::EARL.Report)
  end

  # Each subject is an earl:TestSubject
  subjects.keys.each do |u|
    graph << RDF::Statement.new(u, RDF.type, RDF::Vocab::EARL.TestSubject)
  end

  # Each assertion test is a earl:TestCriterion and earl:TestCase
  test_resources.each do |u|
    graph << RDF::Statement.new(u, RDF.type, RDF::Vocab::EARL.TestCriterion)
    graph << RDF::Statement.new(u, RDF.type, RDF::Vocab::EARL.TestCase)
  end

  raise "Warnings issued in strict mode" if strict && @warnings > 0
end

Instance Attribute Details

#graph ⇒ Object (readonly)

Returns the value of attribute graph.



22
23
24
 
# File 'lib/earl_report.rb', line 22

def graph
  @graph
end

#verbose ⇒ Object (readonly)

Returns the value of attribute verbose.



23
24
25
 
# File 'lib/earl_report.rb', line 23

def verbose
  @verbose
end

Instance Method Details

#generate(format: :html, io: nil, template: nil, **options) ⇒ String

Dump the coalesced output graph

If no io option is provided, the output is returned as a string

Parameters:

  • format (Symbol) (defaults to: :html) —

    (:html)

  • io (IO) (defaults to: nil) —

    (nil) IO to output results

  • options (Hash{Symbol => Object})
  • template (String) (defaults to: nil) —

    HAML template for generating report

Returns:

  • (String) —

    serialized graph, if io is nil



478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
 
# File 'lib/earl_report.rb', line 478

def generate(format: :html, io: nil, template: nil, **options)

  status("generate: #{format}")
  ##
  # Retrieve Hashed information in JSON-LD format
  case format
  when :jsonld, :json
    json = json_hash.to_json(JSON::LD::JSON_STATE)
    io.write(json) if io
    json
  when :turtle, :ttl
    if io
      earl_turtle(io: io)
    else
      io = StringIO.new
      earl_turtle(io: io)
      io.rewind
      io.read
    end
  when :html
    haml = case template
    when String then template
    when IO, StringIO then template.read
    else
      File.read(File.expand_path('../earl_report/views/earl_report.html.haml', __FILE__))
    end

    # Generate HTML report
    html = if Haml.const_defined?(:Template)
      Haml::Template.new(format: :xhtml) {haml}.render(self, tests: json_hash)
    else
      Haml::Engine.new(haml, format: :xhtml).render(self, tests: json_hash)
    end
    if defined?(::HtmlBeautifier)
      html = HtmlBeautifier.beautify(html)
    end
    io.write(html) if io
    html
  else
    writer = RDF::Writer.for(format)
    writer.dump(@graph, io, standard_prefixes: true, **options)
  end
end