Difference between revisions of "Talk:Natural Collections Description"

#!/bin/bash
ncdPrefixes=("ncd.base" "ncd.cd" "ncd.coll" "ncd.colltype" "ncd.com" 
             "ncd.core" "ncd.ct" "ncd.di" "ncd.gr" "ncd.inst" 
             "ncd.itype" "ncd.or" "ncd.ost" "ncd.p" "ncd.proc" 
             "ncd.pub" "ncd.sp" "ncd.spm" "ncd.spmi" "ncd.t" 
             "ncd.tc" "ncd.tn" "ncd.to" "ncd.trank" )
         # TODO unused: "TaxonOccurrenceInteraction.rdf" "TermWithSource.rdf"
ncdFiles=("Base.rdf" "ContactDetails.rdf" "Collection.rdf" "CollectionType.rdf" "Common.rdf" 
          "Core.rdf" "CyclicityTerm.rdf" "DigitalImage.rdf" "GeographicRegion.rdf" "Institution.rdf" 
          "InstitutionType.rdf" "OccurrenceRecord.rdf" "OccurrenceStatusTerm.rdf" "Person.rdf" "Procedure.rdf" 
          "PublicationCitation.rdf" "Specimen.rdf" "SpeciesProfileModel.rdf" "SPMInfoItems.rdf" "Team.rdf" 
          "TaxonConcept.rdf" "TaxonName.rdf" "TaxonOccurrence.rdf" "TaxonRank.rdf")
for (( i=0; i<${#ncdPrefixes[@]}; i++ ));do
  fileName=${ncdFiles[$i]}
  nsPrefix=${ncdPrefixes[$i]}
  printf "%2d: %15s %s …" $((i + 1)) $nsPrefix $fileName
  if [ -e  $fileName ];then
    if ! [ -d  "./export" ];then
      mkdir "./export"
    fi
    gawk --assign glob_coll_namespacePrefix="$nsPrefix" -f parseNCD.awk "${fileName%.*}.rdf" > "./export/${fileName%.*}.mwt"
    printf " (OK)\n"
  else
    printf " (not existing)\n"
  fi
done

#!/bin/awk
# BEGIN {} # block executed before processing
# END {} # block executed after processing
# FS input field separator (default: " ")
# RS record separator (default: newline)

# TODO 
# 2012-12-05 18:13:45: glob_instanceOfClass als value mit rdfs:range innheralb des Klassennamens? 
# 2012-11-27 11:04:50 check instances of Classes if they are assigned to the right concept type or/and instance-relation (skos:Collection or skos:inScheme or vann:termGroup etc)

##########################################
# This file converts NCD Collection.rdf
# into SMW {{#formlink}}. See also in the 
# BEGIN {…} section. An adapted version of
# this file may also be applicable to the 
# other RDF files in NCD scheme
#---- Usage: save this file as NCD-Collection.awk and execute:
# fileName="ContactDetails"; gawk --assign glob_coll_namespacePrefix="ncd.cd" -f parseNCD.awk "NCD-$fileName.rdf" > "NCD-$fileName.mwt"
##########################################


#### programm developer checks ###########
# Check for all elements within range of 
# e.g. owl:DatatypeProperty to owl:DatatypeProperty
#
# sed --quiet "/owl:DatatypeProperty/,/owl:DatatypeProperty/{ s@\t*@@g; s@[ >].*@@g; p;}"  "NCDTest-Collection.rdf" | sort --unique
# sed --quiet "/owl:ObjectProperty/,/owl:ObjectProperty/{ s@\t*@@g; s@[ >].*@@g; p;}"  "NCDTest-Collection.rdf" | sort --unique
# sed --quiet "/owl:Class/,/owl:Class/{ s@\t*@@g; s@[ >].*@@g; p;}"  "NCDTest-Collection.rdf" | sort --unique
# awk '$0~/<[A-Za-z]+ /{ print $0}' "NCDTest-Collection.rdf" | sed --quiet "s@.*\(<[A-Za-z]\+\).*@\1@g;p" | sort --unique
#--- check elements for instances of classes: get all instances in $filename and list all elements in it (just a programm developer check)
#--- just display execution command:
# filename="NCDTest-Collection.rdf";awk '$0~/<[A-Za-z]+ /{ print $0}' "$filename" | sed --quiet "s@.*<\([A-Za-z]\+\).*@\1@g;p" | sort --unique | sed "s@\(.*\)@sed --quiet \"/<\1/,/<\\\/\1/{ s\@\\\t*\@\@g; s\@[ >].*\@\@g; p;}\" \"$filename\" | sort --unique@"
## execute command:
# filename="NCDTest-Collection.rdf";awk '$0~/<[A-Za-z]+ /{ print $0}' "$filename" | sed --quiet "s@.*<\([A-Za-z]\+\).*@\1@g;p" | sort --unique | sed "s@\(.*\)@sed --quiet \"/<\1/,/<\\\/\1/{ s\@\\\t*\@\@g; s\@[ >].*\@\@g; p;}\" \"$filename\" | sort --unique@" | sh | sort --unique

# perl -pne 'BEGIN {undef $/} s/&nbsp;&nbsp;│<!--\n\n-->\n\n<!--\n-->&nbsp;&nbsp; +/\n* /gm'  NCDTest-Collection.mwt > NCDTest-Collection-export.mwt
##########################################
# GLOBAL VARIABLES start with glob_…
##########################################


BEGIN {
  FS ="\"" # field separator to quotation mark " (default is " ")
  if(glob_coll_namespacePrefix=="") {glob_coll_namespacePrefix="ncd.missing"}
}# BEGIN

####### functions start ##################
  function ltrim(s) { sub(/^[ \t]+/, "", s); return s }
  function rtrim(s) { sub(/[ \t]+$/, "", s); return s }
  function trim(s)  { return rtrim(ltrim(s)); }
  
  function get_rdfsLabel() {
    #<rdfs:label>Collection Identifier</rdfs:label> 
    return trim(gensub(/.*<(rdfs:label)>(.+)<\/rdfs:label>.*/, "\\2", "g", $0))
  }# get_rdfsLabel
  
  function get_attributeValueUrlEncoded() {
    # must be FS="\"" !!
    #<rdfs:domain rdf:resource="#Collection"/>
    firstAttributeValue=$2
    if (firstAttributeValue~/^#/) {
      firstAttributeValue=gensub(/^#/, glob_coll_namespaceUriEncoded, "g",  firstAttributeValue)
    } else {
      firstAttributeValue=gensub(/#/, "%23", "g", firstAttributeValue)
    }
    return firstAttributeValue
  }# get_attributeValueUrlEncoded
  
  function get_xmlDataElement (xpathQuery) {
    # use xml_grep and sed to get trimmed single line values
    # "xml_grep xml_grep --text_only --cond \"" xpathQuery "\" " FILENAME " | sed 's@^ *@@;:a;N;$!ba;s/\\n\\t\\+//g;'" | getline output
    # has less features, e.g. get only the attribute
    "xpath " FILENAME " \"" xpathQuery "\"  2>/dev/null | tr --delete '\\n' | sed 's@[ \\t]\\+@ @g;' " | getline output
    #  2>/dev/null removes the xpath's script insertions like "-- NODE --"
    output = gensub(/<a *href *= *['\"]([^'\">]+)['\"][^>]*>([^<]+)<\/a>/,"<nowiki>\\1</nowiki>", "g", output)#"
    output = trim(output)
    output = gensub(/<\/p> *<p>/,"</p>%0A%0A<p>", "g", output)
    output = gensub(/[ \t]*<\/?p>[ \t]*/,"", "g", output)
    output = gensub(/\[\.\.\.\]/,"[…]", "g", output)
    return output
  }

  function get_conceptType(typeUrl){
    
    switch (tolower(gensub(/.*#/,"", "g", typeUrl))) {
     case "datatypeproperty":
         type="data type"
     case "class":
         type="class"
     case "value":
         type="value"
     default:
         type="property"
     }
     return type
  }
  function emptyGlobalVariables() {
    glob_conceptDefinition = "";
    glob_conceptDomain = "";
    glob_conceptID = "";
    glob_conceptNotes = "";
    glob_conceptRange = "";
    glob_conceptType = "";
    glob_rdfs_isDefinedBy = "";
    glob_rdfs_Label = "";
    glob_rdfs_subClassOf = "";
    glob_rdfs_subPropertyOf = "";
    glob_instanceOfClass = "";
  }# emptyGlobalVariables
  
  function create_formLink_Concept_collection() {
    print "* {{#formlink: form=Concept collection\n" \
      " |link text=" glob_coll_title "&nbsp;[[File:Pencil.png|link=]]\n" \
      " |target=" glob_coll_title "\n" \
      " |query string =<!--"
      print "-->Concept collection[concept scheme]=Natural Collections Description<!--"
      if (glob_coll_creators) {print "-->&Concept collection[creators]=" gensub(/ *\/ */, ", ", "g", glob_coll_creators) "<!--" }
      if (glob_coll_description) {print "-->&Concept collection[description]=" glob_coll_description "<!--"}
      if (glob_coll_notes) {print "-->&Concept collection[notes]=" glob_coll_notes "<!--"}
      if (glob_coll_issued) {print "-->&Concept collection[issued]=" glob_coll_issued "<!--"}
      if (glob_coll_modified) {print "-->&Concept collection[modified]=" glob_coll_modified "<!--"}
      if (glob_coll_namespaceUriEncoded) {print "-->&Concept collection[preferred namespace uri]=<nowiki>" glob_coll_namespaceUriEncoded "</nowiki>" "<!--"}
      if (glob_coll_namespacePrefix) {print "-->&Concept collection[preferred namespace prefix]=" glob_coll_namespacePrefix "<!--"}
    print "-->}}\n----\n<!--"
    }
  function create_formLink(conceptID, kindOfConceptComment, startNewOutputSection) {
    # use global variables to generate the form link
    
    #### start ####
    print ((startNewOutputSection == 0) ? "-->&nbsp;&nbsp;" : "-->\n*" ) \
      " {{#formlink: form=Concept\n | link text =" \
      glob_coll_namespacePrefix ":" conceptID "&nbsp;[[File:Pencil.png|link=]]\n | target =" glob_coll_namespacePrefix ":" conceptID "\n | query string =<!--" kindOfConceptComment
    if (glob_rdfs_Label) print "-->Concept[label]=" glob_rdfs_Label "<!--"
    # append all other fields
    if (glob_conceptDefinition) print "-->&Concept[definition]=" trim(glob_conceptDefinition) "<!--"
    if (glob_conceptNotes) print "-->&Concept[notes]=" trim(glob_conceptNotes) "<!--"
    if (glob_conceptType) print "-->&Concept[concept type]=" trim(glob_conceptType) "<!--"
    if (glob_rdfs_isDefinedBy) 
      print "-->&Concept[is defined by]=<nowiki>" trim(glob_rdfs_isDefinedBy) "</nowiki><!--"
    else
      print "-->&Concept[is defined by]=<nowiki>" glob_coll_namespace_isDefinedBy "</nowiki><!--"
    
    #if (glob_conceptDomain || glob_conceptRange) 
    print "-->&Concept scheme relation[1][scheme]=Natural Collections Description<!--"
    # if instance of class then restrict it to rdfs:domain
    i_cr=1 # index concept relation
    if (glob_instanceOfClass) glob_conceptDomain = glob_coll_namespaceUriEncoded glob_instanceOfClass
    if (glob_conceptDomain) print "-->&Concept scheme relation[1][property domain]=<nowiki>" trim(glob_conceptDomain) "</nowiki><!--"
    if (glob_conceptRange)  print "-->&Concept scheme relation[1][property range]=<nowiki>" trim(glob_conceptRange) "</nowiki><!--"
    if (glob_rdfs_subPropertyOf) {print "-->&Concept relation[" i_cr "][relation]=rdfs: subproperty of<!--"}
    if (glob_rdfs_subPropertyOf) {print "-->&Concept relation[" i_cr "][uri]=<nowiki>" trim(glob_rdfs_subPropertyOf) "</nowiki><!--"
      i_cr=i_cr +1
    }
    
    if (glob_rdfs_subClassOf)    {print "-->&Concept relation[" i_cr "][relation]=rdfs: subclass of<!--"}
    if (glob_rdfs_subClassOf)    {print "-->&Concept relation[" i_cr "][uri]=<nowiki>" trim(glob_rdfs_subClassOf) "</nowiki><!--"
      i_cr=i_cr +1
    }
    
    if (glob_coll_title)         {print "-->&Concept relation[" i_cr "][relation]=skos: collection<!--"}
    if (glob_coll_title)         {print "-->&Concept relation[" i_cr "][internal page]=" glob_coll_title "<!--"
      i_cr=i_cr +1
    }
    #### end ####
    print  "-->}}&nbsp;&nbsp;&#x2502;<!--"
    emptyGlobalVariables()
  }# create_formLink
####### functions end ####################

########## Ontology (Collection) ################
/.*<owl:Ontology/,/.*<\/owl:Ontology>/ {
  if ($0~/<owl:Ontology/) {
    glob_coll_namespace_isDefinedBy=$2
    glob_coll_namespaceUriEncoded=$2 "%23"
    glob_coll_namespaceUri=$2 "#"
  }
  if ($0~/<dcterms:issued/) { glob_coll_issued=gensub(/.*([0-9]{4})-([0-9]{2})-([0-9]{2}).*/, "\\1%2F\\2%2F\\3", "g", get_xmlDataElement("*/owl:Ontology/dcterms:issued/text()")) }
  if ($0~/<dcterms:modified/) { glob_coll_modified=gensub(/.*([0-9]{4})-([0-9]{2})-([0-9]{2}).*/, "\\1%2F\\2%2F\\3", "g", get_xmlDataElement("*/owl:Ontology/dcterms:modified/text()")) }
  glob_coll_title = "NCD" gensub(/([A-Z])/, " \\1", "g", gensub(/(NCD-|\.rdf)/,"", "g", FILENAME))
  if ($0~/<dc:publisher/) {}
  if ($0~/<dc:creator/) {glob_coll_creators=get_xmlDataElement("*/owl:Ontology/dc:creator/text()")}
  if ($0~/<dc:description/) {glob_coll_description=get_xmlDataElement("*/owl:Ontology/dc:description/node()")}
  if ($0~/<rdfs:comment/) {glob_coll_notes=get_xmlDataElement("*/owl:Ontology/rdfs:comment/node()")}
  if ($0~/<\/owl:Ontology>/) {
    create_formLink_Concept_collection()
  }
}

########## ObjectProperty ################
# in the range of <owl:ObjectProperty … </owl:ObjectProperty> do:
/.*<owl:ObjectProperty/,/.*<\/owl:ObjectProperty>/ {
  glob_conceptType = "property"
  # <rdfs:comment <rdfs:domain <rdfs:isDefinedBy <rdfs:label <rdfs:range <rdfs:subPropertyOf
  # get glob_conceptID in <owl:ObjectProperty rdf:ID="collectionId">
  if ($0~/.*<owl:ObjectProperty/) { # print $2 # check data
    glob_conceptID=$2
    glob_conceptDefinition = get_xmlDataElement("*/owl:ObjectProperty[@rdf:ID='" glob_conceptID "']/rdfs:comment/node()")
  }
  else {
    if ($0~/<rdfs:label/) { glob_rdfs_Label = get_rdfsLabel() }
    if ($0~/.*<rdfs:domain/) { glob_conceptDomain = get_attributeValueUrlEncoded() }
      # print "domain: " glob_conceptDomain
    if ($0~/.*<rdfs:range/) { glob_conceptRange = get_attributeValueUrlEncoded() }
      # print "range: " glob_conceptRange
    if ($0~/.*<rdfs:isDefinedBy/) { glob_rdfs_isDefinedBy = get_attributeValueUrlEncoded() }
      # print "rdfs:isDefinedBy: " glob_rdfs_isDefinedBy
    if ($0~/.*<rdfs:subPropertyOf/) { glob_rdfs_subPropertyOf = get_attributeValueUrlEncoded() }
      # print "rdfs:subPropertyOf: " glob_rdfs_subPropertyOf
  }# definitions within <owl:ObjectProperty>…</owl:ObjectProperty>

  if ($0~/.*<\/owl:ObjectProperty>/) {
    print create_formLink(glob_conceptID, "ObjectProperty")
  }
}# values in ObjectProperty

########## DatatypeProperty ##############
# in the range of <owl:DatatypeProperty … </owl:DatatypeProperty> do:
/<owl:DatatypeProperty/,/<\/owl:DatatypeProperty>/ {
  glob_conceptType="data type"
  # <rdfs:comment <rdfs:domain <rdfs:isDefinedBy <rdfs:label <rdfs:range <rdfs:subPropertyOf
  # get glob_conceptID in <owl:DatatypeProperty rdf:ID="collectionId">
  if ($0~/.*<owl:DatatypeProperty/) { # print $2 # check data
    glob_conceptID=$2
    glob_conceptDefinition = get_xmlDataElement("*/owl:DatatypeProperty[@rdf:ID='" glob_conceptID "']/rdfs:comment/node()")
      # print "rdfs:comment (definition): " glob_conceptDefinition
  }
  else {
    if ($0~/.*<rdfs:label/) { glob_rdfs_Label = get_rdfsLabel()  }
      # print "rdfs:label: " glob_rdfs_Label
    if ($0~/.*<rdfs:domain/) { glob_conceptDomain = get_attributeValueUrlEncoded() }
      # print "domain: " glob_conceptDomain
    if ($0~/.*<rdfs:range/) { glob_conceptRange = get_attributeValueUrlEncoded() }
    if ($0~/.*<rdf:type/) { glob_conceptType = get_conceptType(get_attributeValueUrlEncoded()) }
      # print "range: " glob_conceptRange
    if ($0~/.*<rdfs:isDefinedBy/) { glob_rdfs_isDefinedBy = get_attributeValueUrlEncoded() }
      # print "rdfs:isDefinedBy: " glob_rdfs_isDefinedBy
    if ($0~/.*<rdfs:subPropertyOf/) { glob_rdfs_subPropertyOf = get_attributeValueUrlEncoded() }
      # print "rdfs:subPropertyOf: " glob_rdfs_subPropertyOf
  }# definitions within <owl:DatatypeProperty>…</owl:DatatypeProperty>

  if ($0~/.*<\/owl:DatatypeProperty>/) {
    #### Create the form link ####
    print create_formLink(glob_conceptID, "DatatypeProperty")
  }
}# values in DatatypeProperty


########## owl:Class and its instances ####
# Classes and instances of a particular class
$0~/<owl:Class/ {# Class definition starts
  classNameID=$2
  readingOwlClass=classNameID
  glob_conceptDefinition = get_xmlDataElement("*/owl:Class[@rdf:ID='" classNameID "']/rdfs:comment/node()")
}# owl:Class

readingOwlClass != "" {
  glob_conceptType = "class"
  if ($0~/<rdfs:label>/) { glob_rdfs_Label = get_rdfsLabel() }
  if ($0~/.*<rdfs:subClassOf/) { glob_rdfs_subClassOf = get_attributeValueUrlEncoded() }
    # print "rdfs:subClassOf: " glob_rdfs_subClassOf
  if ($0~/.*<rdfs:isDefinedBy/) { glob_rdfs_isDefinedBy = get_attributeValueUrlEncoded() }
    # print "rdfs:isDefinedBy: " glob_rdfs_isDefinedBy
  if ($0~/.*<rdf:type/) { glob_conceptType = get_conceptType(get_attributeValueUrlEncoded()) }
}
$0~/.*<\/owl:Class>/ {# Class definition closes
  readingOwlClass=""
  print create_formLink(classNameID, "Class", 1); # separate for output
}

# reading instances of this class
classNameID != "" {
  if(index ($0, "<" classNameID ) > 0) {# an instance of the class was found, e.g. <PrimaryGroupingPrincipleTypeTerm …> of <owl:Class rdf:ID="PrimaryGroupingPrincipleTypeTerm">
    glob_conceptType="value"
    glob_conceptID=$2
    readingInstanceOfClass=classNameID
  }# a class property found
  if(readingInstanceOfClass) {
    if ($0~/.*<rdfs:label/) glob_rdfs_Label = get_rdfsLabel() # get_xmlDataElement("*/" classNameID "[@rdf:ID='" glob_conceptID "']/rdfs:label/node()")
    if ($0~/.*<rdfs:comment/) glob_conceptNotes = get_xmlDataElement("*/" classNameID "[@rdf:ID='" glob_conceptID "']/rdfs:comment/node()")
    if ($0~/.*<dc:title/) { } # don't know yet what to do with it
    if ($0~/.*<base:definition/) glob_conceptDefinition = get_xmlDataElement("*/" classNameID "[@rdf:ID='" glob_conceptID "']/base:definition/node()")
    if ($0~/.*<tbase:definition/) glob_conceptDefinition = get_xmlDataElement("*/" classNameID "[@rdf:ID='" glob_conceptID "']/tbase:definition/node()")
    if ($0~/.*<rdfs:subClassOf/) { glob_rdfs_subClassOf = get_attributeValueUrlEncoded() }
    if ($0~/.*<rdf:type/) { glob_conceptType = get_conceptType(get_attributeValueUrlEncoded()) }
    # no glob_rdfs_subPropertyOf but a glob_instanceOfClass
    glob_instanceOfClass = classNameID
  }
  if(index ($0, "</" classNameID ">") > 0) {# the 
    readingInstanceOfClass=""
    print create_formLink(glob_conceptID, "instance of class " classNameID);
  }
}

# xml_grep --text_only "owl:Ontology/rdfs:comment" NCDTest-Collection.rdf | sed ':a;N;$!ba;s/\n\t\+//g; s@ *\t*\(.\+\) *\t*@\1@g'
# TODO properties in a class