#!/usr/bin/env python3
# encoding: utf-8
"""
Update XML entries
==================
This script converts the job dependency settings in XML files to the new design and creates analysis group
for all test cases of type TestSuiteTestcase.
.. runblock:: console
$ ./contrib/updateXMLEntries.py -h
For the update of the job dependencies the following command line should be sufficient stating the path to the base directory only::
updateXMLEntries.py -i <TRACE_SUITE_PATH>/distrib/testing/standard/level3
If the analysis groups have to be created two additional arguments have to be given. The test case folder has to be the path
to all needed test cases of type TestSuiteTestCase.::
updateXMLEntries.py -i <TRACE_SUITE_PATH>/distrib/testing/standard/level3 --createAnalysisTags --testCaseFolder <PATH>
If only specific XML files are desired the search pattern and/or an ignore pattern can be defined::
updateXMLEntries.py -i <TESTING_PATH> --createAnalysisTags --extension bump_hybrid.xml --testCaseFolder <PATH>
updateXMLEntries.py -i <TESTING_PATH> --createAnalysisTags --ignore TRACE_SUITE*.xml --testCaseFolder <PATH>
The functionality can be tested using the following command in the contrib folder by setting the test case path correctly::
./updateXMLEntries.py -i ../data/orig/ -o ../data/ --createAnalysisTags --testCaseFolder <PATH>
In the folder data/orig two XML files in the old design are stored. As test suite test cases 'cooling' and 'POST_bump_hybrid'
are selected. These two test cases must be present in the specified test case folder.
"""
import argparse
import collections
import fnmatch
from lxml import etree
import re
import glob
import os
import sys
from mojo.bricabrac.fileIO import workingDirectory, ensurePath
from mojo.pavayo import computeData as const
PYTHON_EXTENSION = ".py"
TECPLOT_MACRO_EXTENSION = ".mcr"
DEPENDENCY_JOB_NAME_TAG = "job"
DEPENDENCY_JOB_LIST_NAME_TAG = "subLevel"
DEPENDENCY_JOB_LIST_GROUP_NAME_TAG = "level"
INDENTATION_STEP = " " * 4
[docs]def parseOptions():
"""Parse options to convert test case XML files to new design
:return: command line arguments
:rtype: ArgumentParser
"""
parser = argparse.ArgumentParser(description='Convert test case XML files to new design.')
parser.add_argument('-i', '--input', dest='baseFolder', required=True, metavar='PATH', help='path to base folder.')
parser.add_argument('-o', '--output', dest='outputFolder', metavar='PATH',
help='path to output folder. (If not specified files in input folders will be overwritten.)')
parser.add_argument('--extension', dest='fileExtension', default='*.xml', metavar='PATTERN',
help='Files used matching this extension only.')
parser.add_argument('--ignore', dest='fileIgnore', default='', metavar='PATTERN', help='pattern to ignore files.')
parser.add_argument('--testCaseFolder', dest='testCaseFolder', default='', metavar='PATH', help='path to test cases')
parser.add_argument('--createAnalysisTags', action='store_true', default=False, help='')
args = parser.parse_args()
args.baseFolder = os.path.abspath(os.path.expanduser(args.baseFolder))
if args.outputFolder is None:
args.outputFolder = args.baseFolder
else:
args.outputFolder = os.path.abspath(os.path.expanduser(args.outputFolder))
return args
[docs]def changeDependenciesInElement(xmlElement, jobListGroupTag, jobListTag, jobTag):
"""Convert all job dependencies in the selected job list group tag and job list tag
:param xmlElement: pointer in etree
:param jobListGroupTag: job list group type
:param jobListTag: job list type
:param jobTag: job type
:type xmlElement: etree.Element
:type jobListGroupTag: str
:type jobListTag: str
:type jobTag: str
"""
def replaceDependencyEntry(xmlElement, allJobs, jobListGroupDict, jobListDict, jobDependency, newTail):
"""Replace the dependency entry in the etree for one job
:param xmlElement: pointer in etree
:param jobListGroupDict: job list group dictionary of job
:param jobListDict: job list dictionary of job
:param jobDependency: job dependencies
:param newTail: tail for etree entry
:type xmlElement: etree.Element
:type jobListGroupDict: dict
:type jobListDict: dict
:type jobDependency: list
:type newTail: str
"""
for idd, dependency in enumerate(jobDependency):
if len(dependency) == 0:
newDep = etree.Element('dependency')
jobDep = etree.SubElement(newDep, DEPENDENCY_JOB_NAME_TAG)
jobDep.text = dependency.text
jobDep.tail = newTail + INDENTATION_STEP
if jobDep.text not in jobListDict.keys():
for jobListDepName, jobListDepDict in jobListGroupDict.items():
if jobDep.text in jobListDepDict.keys():
subLevelDep = etree.SubElement(newDep, DEPENDENCY_JOB_LIST_NAME_TAG)
subLevelDep.text = jobListDepName
jobDep.tail = newTail + INDENTATION_STEP + INDENTATION_STEP
subLevelDep.tail = newTail + INDENTATION_STEP
print(f" dependency: {jobDep.text} in {DEPENDENCY_JOB_LIST_NAME_TAG} {jobListDepName}.")
break
else:
for jobListGroupDepName, jobListGroupDepDict in allJobs.items():
for jobListDepName, jobListDepDict in jobListGroupDepDict.items():
if jobDep.text in jobListDepDict.keys():
subLevelDep = etree.SubElement(newDep, DEPENDENCY_JOB_LIST_NAME_TAG)
subLevelDep.text = jobListDepName
levelDep = etree.SubElement(newDep, DEPENDENCY_JOB_LIST_GROUP_NAME_TAG)
levelDep.text = jobListGroupDepName
jobDep.tail = newTail + INDENTATION_STEP + INDENTATION_STEP
subLevelDep.tail = newTail + INDENTATION_STEP + INDENTATION_STEP
levelDep.tail = newTail + INDENTATION_STEP
break
if jobDep.text in jobListDepDict.keys():
break
else:
parent_map = {c: p for p in xmlElement.iter() for c in p}
parentElement = parent_map[dependency]
parentElement.remove(dependency)
parentElement[-1].tail = parentElement[-1].tail[:-4]
print(" dependency: removed empty dependency.")
else:
print(f" dependency: {jobDep.text} is changed to new format.")
dependency.clear()
dependency.tag = "dependency"
dependency.text = newTail + INDENTATION_STEP + INDENTATION_STEP
if idd + 1 == len(jobDependency):
dependency.tail = newTail
else:
dependency.tail = newTail + INDENTATION_STEP
dependency[:] = newDep
else:
print(f" dependency: {dependency.text} is unchanged.")
def buildDefDict():
return collections.defaultdict(buildDefDict)
# dictionary of all dependencies sorted by job list group and job list
allJobs = buildDefDict()
# dictionary to set the correct indentation
jobListTails = buildDefDict()
if jobListGroupTag:
jobListGroupXML = xmlElement.findall(f".//{jobListGroupTag}")
else:
jobListGroupXML = [xmlElement]
for jobListGroup in jobListGroupXML:
jobListGroupName = jobListGroup.find('name')
jobListGroupDict = allJobs[jobListGroupName]
jobListGroupTailsDict = jobListTails[jobListGroupName]
for jobList in jobListGroup.findall(jobListTag):
jobListXmlNameTag = jobList.find('name')
jobListName = jobListXmlNameTag.text
jobListGroupTailsDict[jobListName] = jobListXmlNameTag.tail
jobListDict = jobListGroupDict[jobListName]
for job in jobList.findall(jobTag):
jobName = job.find('name').text
jobListDict[jobName] = job.findall('dependency')
for jobListGroupName, jobListGroupDict in allJobs.items():
if jobListGroupTag:
print(" {tag}: {name}".format(tag=jobListGroupTag, name=jobListGroupName if jobListGroupName is not None else ""))
for jobListName, jobListDict in jobListGroupDict.items():
print(f" {jobListTag}: {jobListName}")
newTail = jobListTails[jobListGroupName][jobListName]
for jobName, jobDependency in jobListDict.items():
print(f" {jobTag}: {jobName}")
replaceDependencyEntry(xmlElement, allJobs, jobListGroupDict, jobListDict, jobDependency, newTail)
[docs]def changeDependencyKeywords(xmlRoot):
"""Convert the dependencies in all occurrences of simulationGroup, restartGroup and jobList
:param xmlRoot: base element
:type xmlRoot: etree.Element
"""
simulationList = xmlRoot.findall('.//simulationGroup')
jobListList = xmlRoot.findall('.//jobList')
if len(simulationList) > 0 or len(jobListList) > 0:
print(" Change dependency tags")
if len(simulationList) > 0:
changeDependenciesInElement(xmlRoot, "simulationGroup", "simulation", "job")
changeDependenciesInElement(xmlRoot, "restartGroup", "simulation", "job")
if len(jobListList) > 0:
changeDependenciesInElement(xmlRoot, "", "jobList", "job")
[docs]def createAnalysisGroup(options, xmlRoot, tcName):
"""Create a default for the post-processing jobs in a TestSuiteTestCase
:param xmlRoot: base element
:param tcName: name of test case
:type xmlRoot: etree.Element
:type tcName: str
"""
def addJob(xmlBaseTag, name, executable, args, jobDependency, indentation):
"""
:param xmlBaseTag: parent element in etree
:param name: name of analysis job
:param executable: used executable of analysis job
:param args: arguments of analysis job
:param jobDependency: dependencies of analysis job
:param indentation: basic tail of elements in etree
:type xmlBaseTag: etree.Element
:type name: str
:type executable: str
:type args: str
:type jobDependency: str
:type indentation: str
"""
xmlTag = etree.SubElement(xmlBaseTag, "job")
xmlTag.text = indentation + INDENTATION_STEP
xmlTag.tail = indentation
pyNameTag = etree.SubElement(xmlTag, "name")
pyNameTag.text = name
pyNameTag.tail = indentation + INDENTATION_STEP
pyExecTag = etree.SubElement(xmlTag, "executable")
pyExecTag.text = executable
pyExecTag.tail = indentation + INDENTATION_STEP
pyArgsTag = etree.SubElement(xmlTag, "args")
pyArgsTag.text = args
if jobDependency == "":
pyArgsTag.tail = indentation
else:
pyArgsTag.tail = indentation + INDENTATION_STEP
newDep = etree.SubElement(xmlTag, "dependency")
newDep.text = indentation + INDENTATION_STEP * 2
newDep.tail = indentation
jobDep = etree.SubElement(newDep, DEPENDENCY_JOB_NAME_TAG)
jobDep.text = jobDependency
jobDep.tail = indentation + INDENTATION_STEP
def findAllJobsRecursively(options, baseFolder, xmlTag, analysisName, indentation):
"""
:param baseFolder: current base folder
:param xmlTag: current parent element in etree
:param analysisName: name of analysis job list
:param indentation: current base tail of element
:type baseFolder: str
:type xmlTag: etree.Element
:type analysisName: str
:type indentation: str
"""
with workingDirectory(baseFolder):
pythonFiles = sorted(glob.glob("*" + PYTHON_EXTENSION), key=str.lower)
tecMacroFiles = sorted(glob.glob("*" + TECPLOT_MACRO_EXTENSION), key=str.lower)
folderList = [f for f in os.listdir(os.curdir) if os.path.isdir(f)]
if len(pythonFiles) > 0 or len(tecMacroFiles) > 0:
analysisTag = etree.SubElement(xmlTag, "analysis")
analysisTag.text = indentation + INDENTATION_STEP * 2
analysisTag.tail = indentation
nameTag = etree.SubElement(analysisTag, "name")
nameTag.text = analysisName
nameTag.tail = indentation + INDENTATION_STEP * 2
prevJobName = ""
for file in pythonFiles:
curJobName = os.path.splitext(file)[0]
print(" " * max(len(indentation), 4) + "generate Python job for" + file)
addJob(analysisTag, curJobName, const.PYTHON_TEMPLATE, file, prevJobName, indentation + INDENTATION_STEP * 2)
prevJobName = curJobName
for file in tecMacroFiles:
curJobName = os.path.splitext(file)[0]
print(" " * max(len(indentation), 4) + "generate Tecplot macor job for" + file)
addJob(analysisTag, curJobName, const.TECPLOT_MACRO_TEMPLATE, file, prevJobName, indentation + INDENTATION_STEP * 2)
prevJobName = curJobName
analysisTag[-1].tail = indentation + INDENTATION_STEP
for folder in folderList:
print(" " * max(len(indentation) - 2, 4) + folder)
findAllJobsRecursively(options, folder, xmlTag, folder, indentation)
if xmlRoot.tag == "testSuiteTestCase":
print(" Create analysis group tag")
baseTail = xmlRoot[-1].tail + INDENTATION_STEP
baseElement = etree.Element("analysisGroup")
baseElement.text = baseTail + INDENTATION_STEP
baseElement.tail = "\n"
findAllJobsRecursively(options, os.path.join(options.testCaseFolder, tcName, "analysis"), baseElement, ".", baseTail)
if len(baseElement) > 0:
xmlRoot[-1].tail = baseTail
xmlRoot.append(baseElement)
[docs]def updateXMLEntries():
"""All job dependency settings in the XML files are replaced with the new design matching the specified pattern
and/or ignore rules. For test cases of type TestSuiteTestcase analysis group can be created as required.
"""
args = parseOptions()
for rootFolder, _, files in os.walk(args.baseFolder):
validFiles = [file for file in files if fnmatch.fnmatch(file, args.fileExtension) and not fnmatch.fnmatch(file, args.fileIgnore)]
for file in sorted(validFiles, key=str.lower):
print(f"Modifying XML file {file}")
xmlFilePath = os.path.join(rootFolder, file)
xmlTree = etree.parse(xmlFilePath)
root = xmlTree.getroot()
# Test Suite Test Cases
if root.tag == "testSuiteTestCase":
xmlTestSuiteTC = [root]
else:
xmlTestSuiteTC = root.findall(".//testSuiteTestCase")
for testSuiteTestcaseXML in xmlTestSuiteTC:
tcName = testSuiteTestcaseXML.find('name').text
print("- " + tcName)
createAnalysisPossible = True if testSuiteTestcaseXML.find("inactive") is None else False
changeDependencyKeywords(testSuiteTestcaseXML)
if args.createAnalysisTags and createAnalysisPossible:
createAnalysisGroup(args, testSuiteTestcaseXML, tcName)
# General Test Cases
if root.tag == "generalTestCase":
xmlgeneralTC = [root]
else:
xmlgeneralTC = root.findall(".//generalTestCase")
for generalTestcaseXML in xmlgeneralTC:
tcName = generalTestcaseXML.find('name').text
print("- " + tcName)
changeDependencyKeywords(generalTestcaseXML)
newXMLpath = xmlFilePath.replace(args.baseFolder, args.outputFolder)
ensurePath(newXMLpath)
xmlTree.write(newXMLpath, pretty_print=True, xml_declaration=True, encoding="UTF-8")
return 0
if __name__ == '__main__':
sys.exit(updateXMLEntries())
