louisdevie/mustangproject
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harmonize the 2 validation methods; added tests

Browse Source
This commit is contained in:
Philip Helger
2024-07-09 17:32:50 +02:00
parent be2d57e3e3
commit f18b194d64
2 changed files with 185 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
validator/src/main/java/org/mustangproject/validator/ZUGFeRDValidator.java
View File

@@ -33,6 +33,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element; import org.w3c.dom.Element;
import org.xml.sax.InputSource; import org.xml.sax.InputSource;
import jakarta.xml.bind.DatatypeConverter;
import jakarta.xml.bind.annotation.adapters.HexBinaryAdapter; import jakarta.xml.bind.annotation.adapters.HexBinaryAdapter;
//abstract class //abstract class
@@ -82,21 +83,18 @@ public class ZUGFeRDValidator {
* @return a xml string with the validation result * @return a xml string with the validation result
*/ */
public String validate(String filename) { public String validate(String filename) {
boolean xmlValidity;
context.clear(); context.clear();
StringBuffer finalStringResult = new StringBuffer(); StringBuilder finalStringResult = new StringBuilder();
SimpleDateFormat isoDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); SimpleDateFormat isoDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = new Date(); Date date = new Date();
startTime = Calendar.getInstance().getTimeInMillis(); startTime = Calendar.getInstance().getTimeInMillis();
try { Path path = Paths.get(filename);
Path path = Paths.get(filename); Path pathFilename = path.getFileName ();
context.setFilename(path.getFileName().toString());// set filename without path if (pathFilename != null)
context.setFilename(pathFilename.toString());// set filename without path
} catch (NullPointerException ex) { else
// ignore context.setFilename(filename);// fallback to provided name
} finalStringResult.append("<validation filename='").append (context.getFilename()).append ("' datetime='").append (isoDF.format(date)).append ("'>");
finalStringResult
.append("<validation filename='" + context.getFilename() + "' datetime='" + isoDF.format(date) + "'>");
boolean isPDF = false; boolean isPDF = false;
byte[] content = null; byte[] content = null;
@@ -110,49 +108,37 @@ public class ZUGFeRDValidator {
PDFValidator pdfv = new PDFValidator(context); PDFValidator pdfv = new PDFValidator(context);
File file = new File(filename); File file = new File(filename);
if (!file.exists()) { if (!file.isFile()) {
context.addResultItem( context.addResultItem(
new ValidationResultItem(ESeverity.fatal, "File not found").setSection(1).setPart(EPart.pdf)); new ValidationResultItem(ESeverity.fatal, "File not found").setSection(1).setPart(EPart.pdf));
} else if (file.length() < 32) { } else if (file.length() < 32) {
// with less than 32 bytes it can not even be a proper XML file // with less than 32 bytes it can not even be a proper XML file
// Except it is "<?xml version='1.0'?><xml/>" LOL
context.addResultItem( context.addResultItem(
new ValidationResultItem(ESeverity.fatal, "File too small").setSection(5).setPart(EPart.pdf)); new ValidationResultItem(ESeverity.fatal, "File too small").setSection(5).setPart(EPart.pdf));
} else { } else {
BigFileSearcher searcher = new BigFileSearcher();
content = Files.readAllBytes(file.toPath()); content = Files.readAllBytes(file.toPath());
XMLValidator xv = new XMLValidator(context); XMLValidator xv = new XMLValidator(context);
if (disableNotices) { if (disableNotices) {
xv.disableNotices(); xv.disableNotices();
} }
byte[] pdfSignature = {'%', 'P', 'D', 'F'}; isPDF = ByteArraySearcher.indexOf(content, new byte[] {'%', 'P', 'D', 'F'}) == 0;
isPDF = searcher.indexOf(file, pdfSignature) == 0;
if (isPDF) { if (isPDF) {
pdfv.setFilename(filename); pdfv.setFilename(filename);
pdfv.setFileContents(content); pdfv.setFileContents(content);
optionsRecognized = true; optionsRecognized = true;
try {
if (!file.exists()) {
context.addResultItem(
new ValidationResultItem(ESeverity.exception, "File " + filename + " not found")
.setSection(1));
}
} catch (IrrecoverableValidationError irx) {
// @todo log
}
finalStringResult.append("<pdf>"); finalStringResult.append("<pdf>");
optionsRecognized = true;
try { try {
pdfv.validate(); pdfv.validate();
sha1Checksum = calcSHA1(new FileInputStream(file)); sha1Checksum = calcSHA1(content);
// Validate PDF // Validate PDF
getPdfValidationResults(finalStringResult, pdfv, xv); getPdfValidationResults(finalStringResult, pdfv, xv);
} catch (IrrecoverableValidationError | FileNotFoundException irx) { } catch (IrrecoverableValidationError irx) {
// @todo log LOGGER.info(irx.getMessage());
} }
finalStringResult.append("</pdf>\n"); finalStringResult.append("</pdf>\n");
@@ -176,16 +162,13 @@ public class ZUGFeRDValidator {
// probably no xml file, sth like SAXParseException content not allowed in prolog // probably no xml file, sth like SAXParseException content not allowed in prolog
// ignore isXML is already false // ignore isXML is already false
// in the tests, this may error-out anyway // in the tests, this may error-out anyway
//ex.printStackTrace(); LOGGER.info("No XML part provided");
} }
if (isXML) { if (isXML) {
pdfValidity = true; pdfValidity = true;
optionsRecognized = true; optionsRecognized = true;
xv.setFilename(filename); xv.setFilename(filename);
if (file.exists()) { sha1Checksum = calcSHA1(content);
sha1Checksum = calcSHA1(Files.newInputStream(file.toPath()));
}
displayXMLValidationOutput = true; displayXMLValidationOutput = true;
@@ -201,7 +184,7 @@ public class ZUGFeRDValidator {
try { try {
xv.validate(); xv.validate();
} catch (IrrecoverableValidationError irx) { } catch (IrrecoverableValidationError irx) {
// @todo log LOGGER.info("The hell");
} }
finalStringResult.append(xv.getXMLResult()); finalStringResult.append(xv.getXMLResult());
finalStringResult.append("</xml>"); finalStringResult.append("</xml>");
@@ -214,7 +197,8 @@ public class ZUGFeRDValidator {
} }
} catch (IrrecoverableValidationError | IOException irx) { } catch (IrrecoverableValidationError | IOException irx) {
// @todo log LOGGER.info(irx.getMessage());
context.setInvalid ();
} finally { } finally {
finalStringResult.append(context.getXMLResult()); finalStringResult.append(context.getXMLResult());
finalStringResult.append("</validation>"); finalStringResult.append("</validation>");
@@ -224,123 +208,129 @@ public class ZUGFeRDValidator {
return formatOutput(finalStringResult, isPDF); return formatOutput(finalStringResult, isPDF);
} }
public String validate(InputStream inputStream, String fileNameOfInputStream) { public String validate(InputStream inputStream, String fileNameOfInputStream) {
boolean xmlValidity; context.clear();
context.clear(); StringBuilder finalStringResult = new StringBuilder();
StringBuffer finalStringResult = new StringBuffer(); SimpleDateFormat isoDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat isoDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Date date = new Date();
Date date = new Date(); startTime = Calendar.getInstance().getTimeInMillis();
startTime = Calendar.getInstance().getTimeInMillis(); context.setFilename(fileNameOfInputStream);// set filename without path
context.setFilename(fileNameOfInputStream);// set filename without path finalStringResult.append("<validation filename='").append(context.getFilename()).append("' datetime='").append(isoDF.format(date)).append("'>");
finalStringResult.append("<validation filename='").append(context.getFilename()).append("' datetime='").append(isoDF.format(date)).append("'>");
boolean isPDF = false; boolean isPDF = false;
byte[] content = new byte[0]; byte[] content = null;
try { try {
if (fileNameOfInputStream == null) { if (fileNameOfInputStream == null) {
optionsRecognized = false; optionsRecognized = false;
context.addResultItem(new ValidationResultItem(ESeverity.fatal, "Filename not specified").setSection(10) context.addResultItem(new ValidationResultItem(ESeverity.fatal, "Filename not specified").setSection(10)
.setPart(EPart.pdf)); .setPart(EPart.pdf));
} }
PDFValidator pdfv = new PDFValidator(context); PDFValidator pdfv = new PDFValidator(context);
if (inputStream == null) { if (inputStream == null) {
context.addResultItem( context.addResultItem(
new ValidationResultItem(ESeverity.fatal, "File not found").setSection(1).setPart(EPart.pdf)); new ValidationResultItem(ESeverity.fatal, "File not found").setSection(1).setPart(EPart.pdf));
} else if (inputStream.available() < 32) { } else if (inputStream.available() < 32) {
// with less then 32 bytes it can not even be a proper XML file // with less than 32 bytes it can not even be a proper XML file
context.addResultItem( // Except it is "<?xml version='1.0'?><xml/>" LOL
new ValidationResultItem(ESeverity.fatal, "File too small").setSection(5).setPart(EPart.pdf)); context.addResultItem(
} else { new ValidationResultItem(ESeverity.fatal, "File too small").setSection(5).setPart(EPart.pdf));
content = IOUtils.toByteArray(inputStream); } else {
isPDF = ByteArraySearcher.contains(content, new byte[]{'%', 'P', 'D', 'F'}); content = IOUtils.toByteArray(inputStream);
XMLValidator xv = new XMLValidator(context); XMLValidator xv = new XMLValidator(context);
if (isPDF) { if (disableNotices) {
pdfv.setFilename(fileNameOfInputStream); xv.disableNotices();
pdfv.setFileContents(content); }
isPDF = ByteArraySearcher.indexOf(content, new byte[] {'%', 'P', 'D', 'F'}) == 0;
if (isPDF) {
// Avoid reading again from file
pdfv.setFilenameAndContents(fileNameOfInputStream, content);
optionsRecognized = true; optionsRecognized = true;
finalStringResult.append("<pdf>"); finalStringResult.append("<pdf>");
try { try {
pdfv.validate(); pdfv.validate();
sha1Checksum = calcSHA1(inputStream); sha1Checksum = calcSHA1(content);
// Validate PDF // Validate PDF
getPdfValidationResults(finalStringResult, pdfv, xv); getPdfValidationResults(finalStringResult, pdfv, xv);
} catch (IrrecoverableValidationError irx) { } catch (IrrecoverableValidationError irx) {
LOGGER.info(irx.getMessage()); LOGGER.info(irx.getMessage());
} }
finalStringResult.append("</pdf>\n"); finalStringResult.append("</pdf>\n");
context.clearCustomXML(); context.clearCustomXML();
} else { } else {
boolean isXML = false; boolean isXML = false;
try { String xmlAsString = null;
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); try {
DocumentBuilder db = dbf.newDocumentBuilder(); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
content = XMLTools.removeBOM(content); content = XMLTools.removeBOM(content);
String s = new String(content, StandardCharsets.UTF_8); xmlAsString = new String(content, StandardCharsets.UTF_8);
InputSource is = new InputSource(new StringReader(s)); InputSource is = new InputSource(new StringReader(xmlAsString));
Document doc = db.parse(is); Document doc = db.parse(is);
Element root = doc.getDocumentElement(); Element root = doc.getDocumentElement();
isXML = true;//no exception so far isXML = true;//no exception so far
} catch (Exception ex) { } catch (Exception ex) {
LOGGER.info("No XML part provided"); // probably no xml file, sth like SAXParseException content not allowed in prolog
} // ignore isXML is already false
if (isXML) { // in the tests, this may error-out anyway
pdfValidity = true; LOGGER.info("No XML part provided");
optionsRecognized = true; }
xv.disableAutoload(); if (isXML) {
xv.setFilename(fileNameOfInputStream); pdfValidity = true;
sha1Checksum = calcSHA1(inputStream); optionsRecognized = true;
xv.setStringContent (xmlAsString);
xv.disableAutoload();
xv.setFilename(fileNameOfInputStream);
sha1Checksum = calcSHA1(content);
displayXMLValidationOutput = true; displayXMLValidationOutput = true;
} else { } else {
optionsRecognized = false; optionsRecognized = false;
context.addResultItem(new ValidationResultItem( context.addResultItem(new ValidationResultItem(ESeverity.exception,
ESeverity.exception, "File does not look like PDF nor XML (contains neither %PDF nor <?xml)").setSection(8));
"File does not look like PDF nor XML (contains neither %PDF nor <?xml)"
).setSection(8));
} }
} }
if ((optionsRecognized) && (displayXMLValidationOutput)) { if ((optionsRecognized) && (displayXMLValidationOutput)) {
finalStringResult.append("<xml>"); finalStringResult.append("<xml>");
try { try {
xv.validate(); xv.validate();
} catch (IrrecoverableValidationError irx) { } catch (IrrecoverableValidationError irx) {
LOGGER.info("The hell"); LOGGER.info("The hell");
} }
finalStringResult.append(xv.getXMLResult()); finalStringResult.append(xv.getXMLResult());
finalStringResult.append("</xml>"); finalStringResult.append("</xml>");
context.clearCustomXML(); context.clearCustomXML();
} }
if ((isPDF) && (!pdfValidity)) { if ((isPDF) && (!pdfValidity)) {
context.setInvalid(); context.setInvalid();
} }
} }
} catch (IrrecoverableValidationError | IOException irx) { } catch (IrrecoverableValidationError | IOException irx) {
LOGGER.info(irx.getMessage()); LOGGER.info(irx.getMessage());
context.setInvalid (); context.setInvalid ();
} finally { } finally {
finalStringResult.append(context.getXMLResult()); finalStringResult.append(context.getXMLResult());
finalStringResult.append("</validation>"); finalStringResult.append("</validation>");
} }
return formatOutput(finalStringResult, isPDF); return formatOutput(finalStringResult, isPDF);
} }
private void getPdfValidationResults(StringBuffer finalStringResult, PDFValidator pdfv, XMLValidator xv) throws IrrecoverableValidationError { private void getPdfValidationResults(StringBuilder finalStringResult, PDFValidator pdfv, XMLValidator xv) throws IrrecoverableValidationError {
finalStringResult.append(pdfv.getXMLResult()); finalStringResult.append(pdfv.getXMLResult());
pdfValidity = context.isValid(); pdfValidity = context.isValid();
@@ -356,7 +346,7 @@ public class ZUGFeRDValidator {
} }
} }
private String formatOutput(StringBuffer finalStringResult, boolean isPDF) { private String formatOutput(StringBuilder finalStringResult, boolean isPDF) {
boolean xmlValidity; boolean xmlValidity;
OutputFormat format = OutputFormat.createPrettyPrint(); OutputFormat format = OutputFormat.createPrettyPrint();
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
@@ -408,7 +398,7 @@ public class ZUGFeRDValidator {
/** /**
* Read the file and calculate the SHA-1 checksum * Read the file and calculate the SHA-1 checksum
* *
* @param inputStream the InputStream to read * @param data the InputStream to read
* @return the hex representation of the SHA-1 using uppercase chars * @return the hex representation of the SHA-1 using uppercase chars
* @throws FileNotFoundException if the file does not exist, is a directory * @throws FileNotFoundException if the file does not exist, is a directory
* rather than a regular file, or for some * rather than a regular file, or for some
@@ -416,26 +406,19 @@ public class ZUGFeRDValidator {
* @throws IOException if an I/O error occurs * @throws IOException if an I/O error occurs
* @throws NoSuchAlgorithmException should never happen * @throws NoSuchAlgorithmException should never happen
*/ */
private static String calcSHA1(InputStream inputStream) { private static String calcSHA1(byte[] data) {
MessageDigest sha1 = null; MessageDigest sha1 = null;
try { try {
sha1 = MessageDigest.getInstance("SHA-1"); sha1 = MessageDigest.getInstance("SHA-1");
byte[] buffer = new byte[8192]; sha1.update(data, 0, data.length);
int len = inputStream.read(buffer); } catch (NoSuchAlgorithmException e) {
while (len != -1) {
sha1.update(buffer, 0, len);
len = inputStream.read(buffer);
}
inputStream.close();
} catch (IOException | NoSuchAlgorithmException e) {
LOGGER.error(e.getMessage(), e); LOGGER.error(e.getMessage(), e);
} }
if (sha1 == null) { if (sha1 == null) {
return ""; return "";
} else { } else {
return new HexBinaryAdapter().marshal(sha1.digest()); return DatatypeConverter.printHexBinary(sha1.digest());
} }
} }

55
validator/src/test/java/org/mustangproject/validator/ZUGFeRDValidatorTest.java
View File

@@ -1,6 +1,8 @@
package org.mustangproject.validator; package org.mustangproject.validator;
import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.xmlunit.builder.Input; import org.xmlunit.builder.Input;
@@ -83,6 +85,59 @@ public class ZUGFeRDValidatorTest extends ResourceCase {
} }
public void testPDFValidationInputStream() {
byte[] fileBytes = getResourceAsByteArray("invalidPDF.pdf");
/**used to be Rule Status
Specification: ISO 19005-3:2012, Clause: 6.2.11.4, Test number: 4
If the FontDescriptor dictionary of an embedded CID font contains a CIDSet stream, then it shall identify all CIDs which are present in the font program, regardless of whether a CID in the font is referenced or used by the PDF or not. Failed
2 occurrences Hide
PDCIDFont
fontFile_size == 0 || fontName.search(/[A-Z]{6}\+/) != 0 || CIDSet_size == 0 || cidSetListsAllGlyphs == true
root/document[0]/pages[1](9 0 obj PDPage)/contentStream[0](18 0 obj PDContentStream)/operators[166]/font[0](WIUIIO+CIDFont+F2)/DescendantFonts[0](WIUIIO+CIDFont+F2)
root/document[0]/pages[1](9 0 obj PDPage)/contentStream[0](18 0 obj PDContentStream)/operators[192]/font[0](VEXQUA+CIDFont+F1)/DescendantFonts[0](VEXQUA+CIDFont+F1)
but new sample since that has been downgraded to warning
*/
ZUGFeRDValidator zfv = new ZUGFeRDValidator();
String res = zfv.validate(new ByteArrayInputStream (fileBytes), "invalidPDF.pdf");
assertThat(res).valueByXPath("/validation/pdf/summary/@status")
.isEqualTo("invalid");
assertThat(res).valueByXPath("/validation/xml/summary/@status")
.isEqualTo("valid");
assertThat(res).valueByXPath("/validation/summary/@status")
.isEqualTo("invalid");
fileBytes = getResourceAsByteArray("validAvoir_FR_type380_BASICWL.pdf");
zfv = new ZUGFeRDValidator();
res = zfv.validate(new ByteArrayInputStream (fileBytes), "validAvoir_FR_type380_BASICWL.pdf");
assertThat(res).valueByXPath("/validation/summary/@status")
.isEqualTo("valid");
fileBytes = getResourceAsByteArray("validXRechnung.pdf");
zfv = new ZUGFeRDValidator();
res = zfv.validate(new ByteArrayInputStream (fileBytes), "validXRechnung.pdf");
assertThat(res).valueByXPath("/validation/summary/@status")
.isEqualTo("valid");
fileBytes = getResourceAsByteArray("invalidXRechnung.pdf");
zfv = new ZUGFeRDValidator();
res = zfv.validate(new ByteArrayInputStream (fileBytes), "invalidXRechnung.pdf");
assertThat(res).valueByXPath("/validation/summary/@status")
.isEqualTo("invalid");
zfv = new ZUGFeRDValidator();
res = zfv.validate(new ByteArrayInputStream (new byte[0]), "/does/not/exist");
assertThat(res).valueByXPath("/validation/summary/@status")
.isEqualTo("invalid");
}
/*** /***
* the XMLValidatorTests only cover the <xml></xml> part, this one includes the root element and * the XMLValidatorTests only cover the <xml></xml> part, this one includes the root element and
* the global <summary></summary> part as well * the global <summary></summary> part as well