Merge branch 'master' into position_accounting_reference

History.md

@@ -1,22 +1,32 @@
-#558
+2.16.3
-#686
+=======
-#739
+2025-03-03
+- #558 ZUGFeRDInvoiceImporter does not read BankDetails.accountName
+- #686 Item: add BillingSpecifiedPeriod
+- #739 also parse invoiceperiod from ubl
+- #745 be able to specify legalorganisation id without schema
+- #747 correct profile detection
+- #710 Validation Error due to empty elements
+- #712 Correct bracket setting on condition for output of allowance reason.
+- #725 Unable to perform XML-oriented attacks
+- #685 Security Issue: XXE Vulnerability in ZUGFeRDInvoiceImporter (PR #725)
+- #761 Allow to set item allowance/charges from JSON
 
 2.16.2
 =======
 2025-02-04
--#705 specifiedLogisticsCharge is not imported
+- #705 specifiedLogisticsCharge is not imported
--#707 invoiceimporter may fail if certain values are not set
+- #707 invoiceimporter may fail if certain values are not set
--#708 embedded files cannot be determined
+- #708 embedded files cannot be determined
--#709 ZUGFeRDInvoiceImporter ignored "first" embedded file in list of pdf attachments
+- #709 ZUGFeRDInvoiceImporter ignored "first" embedded file in list of pdf attachments
--#607 Enable flexible PaymentReference and a DocumentName.
+- #607 Enable flexible PaymentReference and a DocumentName.
--#649 Reuse toPDF method to work without any dependencies to the file system
+- #649 Reuse toPDF method to work without any dependencies to the file system
--#650 Add net.sf.offo:fop-hyph
+- #650 Add net.sf.offo:fop-hyph
--#665 Fix #632: Return ubl_creditnote as Standard for CreditNotes
+- #665 Fix #632: Return ubl_creditnote as Standard for CreditNotes
--#684 Optimize validation-report to pdf functionality
+- #684 Optimize validation-report to pdf functionality
--#703 Fill TaxExemptionReason during InvoiceImport.
+- #703 Fill TaxExemptionReason during InvoiceImport.
--#701 Ensure Base64 decoding can handle newlines when decoding a FileAttachment
+- #701 Ensure Base64 decoding can handle newlines when decoding a FileAttachment
--#691 Fix current check failures.
+- #691 Fix current check failures.
 
 2.16.1
 =======

Mustang-CLI/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.mustangproject</groupId>
         <artifactId>core</artifactId>
-        <version>2.16.3-SNAPSHOT</version>
+        <version>2.16.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.mustangproject</groupId>
@@ -12,7 +12,7 @@
         should also work for XRechnung/CII.
     </name>
     <packaging>jar</packaging>
-    <version>2.16.3-SNAPSHOT</version>
+    <version>2.16.4-SNAPSHOT</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.compilerVersion>11</maven.compiler.compilerVersion>
@@ -23,7 +23,7 @@
         <dependency>
             <groupId>org.mustangproject</groupId>
             <artifactId>validator</artifactId>
-            <version>2.16.3-SNAPSHOT</version>
+            <version>2.16.4-SNAPSHOT</version>
             <!-- prototypes of new mustangproject versions can be installed by referring to them and installed to the local repo from a jar file with
            mvn install:install-file -Dfile=mustang-1.5.4-SNAPSHOT.jar -DgroupId=org.mustangproject.ZUGFeRD -DartifactId=mustang -Dversion=1.5.4 -Dpackaging=jar -DgeneratePom=true
            -->

library/pom.xml

@@ -3,13 +3,13 @@
     <parent>
         <groupId>org.mustangproject</groupId>
         <artifactId>core</artifactId>
-        <version>2.16.3-SNAPSHOT</version>
+        <version>2.16.4-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.mustangproject</groupId>
     <artifactId>library</artifactId>
-    <version>2.16.3-SNAPSHOT</version>
+    <version>2.16.4-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>Library to write, read and validate e-invoices (Factur-X, ZUGFeRD, Order-X, XRechnung/CII)</name>
     <description>FOSS Java library to read, write and validate european electronic invoices and orders in the UN/CEFACT

library/src/main/java/org/mustangproject/Item.java

@@ -369,6 +369,29 @@ public class Item implements IZUGFeRDExportableItem {
 			return Allowances.toArray(new IZUGFeRDAllowanceCharge[0]);
 	}
 
+	/***
+	 * jackson convenience method
+	 */
+	public void setItemAllowances(ArrayList<Allowance> theAllowances) {
+		if (theAllowances!=null) {
+			Allowances.clear();
+			for (Allowance theAllowance : theAllowances) {
+				Allowances.add(theAllowance);
+			}
+		}
+	}
+	/***
+	 * jackson convenience method
+	 */
+	public void setItemCharges(ArrayList<Charge> theCharges) {
+		if (theCharges!=null) {
+			Charges.clear();
+			for (Charge theCharge : theCharges) {
+				Charges.add(theCharge);
+			}
+		}
+	}
+
 	@Override
 	public IZUGFeRDAllowanceCharge[] getItemCharges() {
 		if (Charges.isEmpty()) {

library/src/main/java/org/mustangproject/LegalOrganisation.java

@@ -23,6 +23,10 @@ public class LegalOrganisation implements IZUGFeRDLegalOrganisation {
 		this.schemedID = new SchemedID(scheme, ID);
 	}
 
+	public LegalOrganisation(String ID) {
+		this.schemedID = new SchemedID(null, ID);
+	}
+
 	public LegalOrganisation(SchemedID schemedID, String tradingBusinessName) {
 		this.schemedID = schemedID;
 		this.tradingBusinessName=tradingBusinessName;

library/src/main/java/org/mustangproject/XMLTools.java

@@ -130,6 +130,9 @@ public class XMLTools extends XMLWriter {
 	 */
 	public static Date tryDate(String toParse) {
 		SimpleDateFormat formatter = null;
+		if (toParse==null) {
+			return null;
+		}
 		if (toParse.contains("-")) {
 			// from ubl
 			 formatter = new SimpleDateFormat("yyyy-MM-dd");

library/src/main/java/org/mustangproject/ZUGFeRD/ValidationLogVisualizer.java

@@ -1,5 +1,13 @@
 package org.mustangproject.ZUGFeRD;
 
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringReader;
 import org.apache.fop.apps.*;
 import org.apache.fop.apps.io.ResourceResolverFactory;
 import org.apache.fop.configuration.Configuration;
@@ -14,7 +22,7 @@ import javax.xml.transform.*;
 import javax.xml.transform.sax.SAXResult;
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
-import java.io.*;
+
 import java.nio.charset.StandardCharsets;
 
 public class ValidationLogVisualizer {
@@ -70,7 +78,7 @@ public class ValidationLogVisualizer {
 		return baos.toString(StandardCharsets.UTF_8);
 	}
 
-	public void toPDF(String xmlLogfileContent, String pdfFilename) {
+	public byte[] createPDFBytes(String xmlLogfileContent) {
 
 		// the writing part
 
@@ -111,7 +119,8 @@ public class ValidationLogVisualizer {
 // Step 2: Set up output stream.
 // Note: Using BufferedOutputStream for performance reasons (helpful with FileOutputStreams).
 
-		try (OutputStream out = new BufferedOutputStream(new FileOutputStream(pdfFilename))) {
+		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+		try (OutputStream out = new BufferedOutputStream(baos)) {
 
 			// Step 3: Construct fop with desired output format
 			Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, userAgent, out);
@@ -133,6 +142,20 @@ public class ValidationLogVisualizer {
 		} catch (FOPException | IOException | TransformerException e) {
 			LOGGER.error("Failed to create PDF", e);
 		}
+		return baos.toByteArray();
+	}
+
+	public byte[] toPDF(String xmlLogfileContent) {
+		return createPDFBytes(xmlLogfileContent);
+	}
+
+	public void toPDF(String xmlLogfileContent, String pdfFilename) {
+		byte[] pdfData = createPDFBytes(xmlLogfileContent);
+		try (FileOutputStream fos = new FileOutputStream(pdfFilename)) {
+			fos.write(pdfData);
+		} catch (IOException e) {
+			LOGGER.error("Failed to write PDF to file", e);
+		}
 	}
 
 	private static class ClasspathResourceURIResolver implements URIResolver {

library/src/main/java/org/mustangproject/ZUGFeRD/ZUGFeRD2PullProvider.java

@@ -149,7 +149,12 @@ public class ZUGFeRD2PullProvider implements IXMLProvider {
 				if (profile == Profiles.getByName("Minimum")) {
 					xml += "<ram:ID>" + XMLTools.encodeXML(party.getLegalOrganisation().getSchemedID().getID()) + "</ram:ID>";
 				} else {
-					xml += "<ram:ID schemeID=\"" + XMLTools.encodeXML(party.getLegalOrganisation().getSchemedID().getScheme()) + "\">" + XMLTools.encodeXML(party.getLegalOrganisation().getSchemedID().getID()) + "</ram:ID>";
+					String schemeAttribute="";
+					if ((party.getLegalOrganisation().getSchemedID().getScheme()!=null)&&(party.getLegalOrganisation().getSchemedID().getScheme().length()>0)) {
+						schemeAttribute="schemeID=\"" + XMLTools.encodeXML(party.getLegalOrganisation().getSchemedID().getScheme())+"\"";
+
+					}
+					xml += "<ram:ID "+schemeAttribute+">" + XMLTools.encodeXML(party.getLegalOrganisation().getSchemedID().getID()) + "</ram:ID>";
 				}
 			}
 			if (party.getLegalOrganisation().getTradingBusinessName() != null) {
@@ -277,7 +282,7 @@ public class ZUGFeRD2PullProvider implements IXMLProvider {
 		}
 
 		String reason = "";
-		if ((allowance.getReason() != null) && (profile == Profiles.getByName("Extended") || profile == Profiles.getByName("XRechnung")) || profile == Profiles.getByName("EN16931")) {
+		if ((allowance.getReason() != null) && (profile == Profiles.getByName("Extended") || profile == Profiles.getByName("XRechnung") || profile == Profiles.getByName("EN16931"))) {
 			reason = "<ram:Reason>" + XMLTools.encodeXML(allowance.getReason()) + "</ram:Reason>";
 		}
 		String reasonCode = "";

library/src/main/java/org/mustangproject/ZUGFeRD/ZUGFeRDImporter.java

@@ -86,6 +86,7 @@ public class ZUGFeRDImporter extends ZUGFeRDInvoiceImporter {
 			case "urn:factur-x.eu:1p0:minimum":
 				return "MINIMUM";
 			case "urn:ferd:CrossIndustryDocument:invoice:1p0:extended":
+			case "urn:cen.eu:en16931:2017#conformant#urn:zugferd.de:2p0:extended":
 			case "urn:cen.eu:en16931:2017#conformant#urn:factur-x.eu:1p0:extended":
 				return "EXTENDED";
 			default:

library/src/main/java/org/mustangproject/ZUGFeRD/ZUGFeRDInvoiceImporter.java

@@ -1,5 +1,6 @@
 package org.mustangproject.ZUGFeRD;
 
+import javax.xml.XMLConstants;
 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -258,9 +259,14 @@ public class ZUGFeRDInvoiceImporter {
 	}
 
 	private void setDocument() throws ParserConfigurationException, IOException, SAXException, ParseException {
-		final DocumentBuilderFactory xmlFact = DocumentBuilderFactory.newInstance();
+		final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-		xmlFact.setNamespaceAware(true);
+		dbf.setNamespaceAware(true);
-		final DocumentBuilder builder = xmlFact.newDocumentBuilder();
+		dbf.setExpandEntityReferences(false);
+		dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+		dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+		dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+		dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+		final DocumentBuilder builder = dbf.newDocumentBuilder();
 		final ByteArrayInputStream is = new ByteArrayInputStream(rawXML);
 		///    is.skip(guessBOMSize(is));
 		document = builder.parse(is);

library/src/main/java/org/mustangproject/ZUGFeRD/ZUGFeRDVisualizer.java

@@ -21,6 +21,8 @@
 package org.mustangproject.ZUGFeRD;
 
 import com.helger.commons.io.stream.StreamHelper;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.ParserConfigurationException;
 import org.apache.commons.io.IOUtils;
 import org.apache.fop.apps.*;
 import org.apache.fop.apps.io.ResourceResolverFactory;
@@ -90,7 +92,8 @@ public class ZUGFeRDVisualizer {
 	 * @param fis inputstream (will be consumed)
 	 * @return (facturx = cii)
 	 */
-	private EStandard findOutStandardFromRootNode(InputStream fis) {
+	private EStandard findOutStandardFromRootNode(InputStream fis)
+		throws ParserConfigurationException {
 
 		String zf1Signature = "CrossIndustryDocument";
 		String zf2Signature = "CrossIndustryInvoice";
@@ -100,6 +103,11 @@ public class ZUGFeRDVisualizer {
 
 		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
 		dbf.setNamespaceAware(true);
+		dbf.setExpandEntityReferences(false);
+		dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+		dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+		dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+		dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
 		try {
 			DocumentBuilder db = dbf.newDocumentBuilder();
 			Document doc = db.parse(new InputSource(fis));
@@ -121,12 +129,14 @@ public class ZUGFeRDVisualizer {
 		return null;
 	}
 
-	public String visualize(String xmlFilename, Language lang) throws IOException, TransformerException {
+	public String visualize(String xmlFilename, Language lang)
+		throws IOException, TransformerException, ParserConfigurationException {
 		FileInputStream fis = new FileInputStream(xmlFilename);
 		return visualize(fis, lang);
 	}
 
-	public String visualize(InputStream inputXml, Language lang) throws IOException, TransformerException {
+	public String visualize(InputStream inputXml, Language lang)
+		throws IOException, TransformerException, ParserConfigurationException {
 		initTemplates(lang);
 
 		String fileContent = new String(IOUtils.toByteArray(inputXml), StandardCharsets.UTF_8);
@@ -211,7 +221,7 @@ public class ZUGFeRDVisualizer {
 	}
 
 	protected String toFOP(String xmlFilename)
-		throws IOException, TransformerException {
+		throws IOException, TransformerException, ParserConfigurationException {
 
 		FileInputStream fis = new FileInputStream(xmlFilename);
 		EStandard theStandard = findOutStandardFromRootNode(fis);
@@ -264,7 +274,7 @@ public class ZUGFeRDVisualizer {
 			 */
 		try {
 			fopInput = this.toFOP(XMLinputFile.getAbsolutePath());
-		} catch (TransformerException | IOException e) {
+		} catch (TransformerException | IOException | ParserConfigurationException e) {
 			LOGGER.error("Failed to apply FOP", e);
 		}
 		
@@ -291,7 +301,7 @@ public class ZUGFeRDVisualizer {
 			fis = new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8));//rewind :-(
 			
 			fopInput = toFOP(fis, theStandard);
-		} catch (TransformerException | IOException e) {
+		} catch (TransformerException | IOException | ParserConfigurationException e) {
 			LOGGER.error("Failed to apply FOP", e);
 		}
 

library/src/test/java/org/mustangproject/ZUGFeRD/VisualizationTest.java

@@ -20,6 +20,7 @@
  */
 package org.mustangproject.ZUGFeRD;
 
+import javax.xml.parsers.ParserConfigurationException;
 import org.junit.FixMethodOrder;
 import org.junit.runners.MethodSorters;
 import org.mustangproject.ZUGFeRD.ZUGFeRDVisualizer.Language;
@@ -76,9 +77,10 @@ public class VisualizationTest extends ResourceCase {
 			fail("TransformerException should not happen: " + e.getMessage());
 		} catch (IOException e) {
 			fail("IOException should not happen: " + e.getMessage());
+		} catch (ParserConfigurationException e) {
+			fail("ParserConfigurationException should not happen: " + e.getMessage());
 		}
 
-
 		assertNotNull(result);
         /* remove file endings so that tests can also pass after checking
 		   out from git with arbitrary options (which may include CSRF changes)

library/src/test/java/org/mustangproject/ZUGFeRD/XRTest.java

@@ -52,12 +52,17 @@ public class XRTest extends TestCase {
 		TradeParty recipient = new TradeParty("Franz Müller", "teststr.12", "55232", "Entenhausen", "DE");
 		recipient.setEmail("quack@ducktown.org");
 		Invoice i = createInvoice(recipient);
-
+		String legalOrgID="aCustomSellerLegalOrgId";
+		String sellerID="aSellerTradePartyID";
+		i.getSender().setLegalOrganisation(new LegalOrganisation(legalOrgID));
+		i.getSender().setID(sellerID);
 		ZUGFeRD2PullProvider zf2p = new ZUGFeRD2PullProvider();
 		zf2p.setProfile(Profiles.getByName("XRechnung"));
 		zf2p.generateXML(i);
 		String theXML = new String(zf2p.getXML(), StandardCharsets.UTF_8);
 		assertTrue(theXML.contains("<rsm:CrossIndustryInvoice"));
+		assertTrue(theXML.contains("<ram:ID>"+sellerID+"</ram:ID>"));// must be possible without scheme #
+		assertTrue(theXML.contains("<ram:ID>"+legalOrgID+"</ram:ID>"));// must be possible without scheme #
 		assertThat(theXML).valueByXPath("count(//*[local-name()='IncludedSupplyChainTradeLineItem'])")
 			.asInt()
 			.isEqualTo(1); //2 errors are OK because there is a known bug

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.mustangproject</groupId>
     <artifactId>core</artifactId>
-    <version>2.16.3-SNAPSHOT</version>    <packaging>pom</packaging>
+    <version>2.16.4-SNAPSHOT</version>    <packaging>pom</packaging>
     <name>Mustang</name>
 
     <modules>

validator/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.mustangproject</groupId>
         <artifactId>core</artifactId>
-        <version>2.16.3-SNAPSHOT</version>
+        <version>2.16.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.mustangproject</groupId>
@@ -11,7 +11,7 @@
     <name>Library to validate e-invoices (ZUGFeRD, Factur-X and Xrechnung)</name>
 
     <packaging>jar</packaging>
-    <version>2.16.3-SNAPSHOT</version>
+    <version>2.16.4-SNAPSHOT</version>
     <repositories>
         <repository>
             <!-- for jargs -->
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>library</artifactId>
-            <version>2.16.3-SNAPSHOT</version>
+            <version>2.16.4-SNAPSHOT</version>
         </dependency>
 		<dependency>
 			<groupId>org.dom4j</groupId>

validator/src/main/java/org/mustangproject/validator/XMLValidator.java

@@ -10,6 +10,7 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Calendar;
 
+import javax.xml.XMLConstants;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.transform.stream.StreamSource;
@@ -151,6 +152,11 @@ public class XMLValidator extends Validator {
 				final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
 				dbf.setNamespaceAware(true); // otherwise we can not act namespace independently, i.e. use
 				// document.getElementsByTagNameNS("*",...
+				dbf.setExpandEntityReferences(false);
+				dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+				dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+				dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+				dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
 
 				final DocumentBuilder db = dbf.newDocumentBuilder();
 				final InputSource is = new InputSource(new StringReader(zfXML));

validator/src/main/java/org/mustangproject/validator/ZUGFeRDValidator.java

@@ -17,6 +17,7 @@ import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
 
+import javax.xml.XMLConstants;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
@@ -142,6 +143,12 @@ public class ZUGFeRDValidator {
 					String xmlAsString = null;
 					try {
 						DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+						dbf.setNamespaceAware(true);
+						dbf.setExpandEntityReferences(false);
+						dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+						dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+						dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+						dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
 						DocumentBuilder db = dbf.newDocumentBuilder();
 
 						content = XMLTools.removeBOM(content);