Merge pull request #102 from gunnsth/release/v3.0.3

Prepare release v3.0.3

Author: gunnsth

Gopkg.lock

@@ -15,6 +15,7 @@ node {
     env.UNIDOC_PASSTHROUGH_TESTDATA="/home/jenkins/corpus/unidoc-e2e-testdata"
     env.UNIDOC_ALLOBJECTS_TESTDATA="/home/jenkins/corpus/unidoc-e2e-testdata"
     env.UNIDOC_SPLIT_TESTDATA="/home/jenkins/corpus/unidoc-e2e-split-testdata"
+    env.UNIDOC_FDFMERGE_TESTDATA="/home/jenkins/corpus/fdfmerge-testdata"
     env.UNIDOC_GS_BIN_PATH="/usr/bin/gs"
     // Hack for 1.11.5 testing work.
     env.CGO_ENABLED="0"

Jenkinsfile

@@ -12,11 +12,11 @@ import (
 
 const releaseYear = 2019
 const releaseMonth = 6
-const releaseDay = 11
-const releaseHour = 22
+const releaseDay = 27
+const releaseHour = 20
 const releaseMin = 10
 
 // Version holds version information, when bumping this make sure to bump the released at stamp also.
-const Version = "3.0.2"
+const Version = "3.0.3"
 
 var ReleasedAt = time.Date(releaseYear, releaseMonth, releaseDay, releaseHour, releaseMin, 0, 0, time.UTC)

common/version.go

@@ -768,6 +768,7 @@ func (parser *PdfParser) parseXrefTable() (*PdfObjectDictionary, error) {
 	curObjNum := -1
 	secObjects := 0
 	insideSubsection := false
+	unmatchedContent := ""
 	for {
 		parser.skipSpaces()
 		_, err := parser.reader.Peek(1)
@@ -781,13 +782,24 @@ func (parser *PdfParser) parseXrefTable() (*PdfObjectDictionary, error) {
 		}
 
 		result1 := reXrefSubsection.FindStringSubmatch(txt)
+		if len(result1) == 0 {
+			// Try to match invalid subsection beginning lines from previously
+			// read, unidentified lines. Covers cases in which the object number
+			// and the number of entries in the subsection are not on the same line.
+			tryMatch := len(unmatchedContent) > 0
+			unmatchedContent += txt + "\n"
+			if tryMatch {
+				result1 = reXrefSubsection.FindStringSubmatch(unmatchedContent)
+			}
+		}
 		if len(result1) == 3 {
 			// Match
 			first, _ := strconv.Atoi(result1[1])
 			second, _ := strconv.Atoi(result1[2])
 			curObjNum = first
 			secObjects = second
 			insideSubsection = true
+			unmatchedContent = ""
 			common.Log.Trace("xref subsection: first object: %d objects: %d", curObjNum, secObjects)
 			continue
 		}
@@ -801,6 +813,7 @@ func (parser *PdfParser) parseXrefTable() (*PdfObjectDictionary, error) {
 			first, _ := strconv.ParseInt(result2[1], 10, 64)
 			gen, _ := strconv.Atoi(result2[2])
 			third := result2[3]
+			unmatchedContent = ""
 
 			if strings.ToLower(third) == "n" && first > 1 {
 				// Object in use in the file!  Load it.
@@ -829,6 +842,7 @@ func (parser *PdfParser) parseXrefTable() (*PdfObjectDictionary, error) {
 			curObjNum++
 			continue
 		}
+
 		if (len(txt) > 6) && (txt[:7] == "trailer") {
 			common.Log.Trace("Found trailer - %s", txt)
 			// Sometimes get "trailer << ...."
@@ -1521,6 +1535,11 @@ func (parser *PdfParser) ParseIndirectObject() (PdfObject, error) {
 				return &indirect, err
 			}
 			common.Log.Trace("Parsed object ... finished.")
+		} else if bb[0] == ']' {
+			// ']' not used as an array object ending marker, or array object
+			// terminated multiple times. Discarding the character.
+			common.Log.Debug("WARNING: ']' character not being used as an array ending marker. Skipping.")
+			parser.reader.Discard(1)
 		} else {
 			if bb[0] == 'e' {
 				lineStr, err := parser.readTextLine()

core/parser.go

@@ -0,0 +1,169 @@
+/*
+ * This file is subject to the terms and conditions defined in
+ * file 'LICENSE.md', which is part of this source code package.
+ */
+
+package e2etest
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/unidoc/unipdf/v3/annotator"
+	"github.com/unidoc/unipdf/v3/fdf"
+	"github.com/unidoc/unipdf/v3/model"
+)
+
+// FDF merge tests merge FDF data into template PDF data and flattens to an output PDF file.
+// Output files are checked with ghostscript and memory consumption is measured.
+// Set environment variables:
+//		UNIDOC_E2E_FORCE_TESTS to "1" to force the tests to execute.
+//		UNIDOC_FDFMERGE_TESTDATA to the path of the corpus folder.
+//		UNIDOC_GS_BIN_PATH to the path of the ghostscript binary (gs) for validation.
+var (
+	fdfMergeCorpusFolder = os.Getenv("UNIDOC_FDFMERGE_TESTDATA")
+)
+
+// fdfMergeHashes defines a list of known output hashes to ensure that the output is constant.
+// If there is a change in hash need to find out why and update only if the change is accepted.
+var fdfMergeHashes = map[string]string{
+	"NW_null_Business_V04.fdf":      "6e33f219994e4b9ee1e1843c976504df",
+	"NW_null_Business_V05.fdf":      "ff1f8bd39f9be9844a6d85bafe07c790",
+	"NW_null_Business_V05.v1.2.fdf": "ff1f8bd39f9be9844a6d85bafe07c790",
+	"NW_null_Contract_V04.fdf":      "a54f4b42dc34997cfb701ef647cdbdfe",
+	"N_null_Contract.fdf":           "c173340d6492984532cf51a4f5ceb4b6",
+	"Network_Contract_V01.fdf":      "0ae2537bf8a8366aa97c1ca965b88d1f",
+	"checkmark_check.fdf":           "8892cdb01318421f8d198233b80ab8e3",
+	"checkmark_circle.fdf":          "3b1e6ef6aae2a7497b090e0960d2c163",
+	"checkmark_cross.fdf":           "6b16b6d7437a3f59a7e9e72c1ecfd59b",
+	"checkmark_diamond.fdf":         "123488e428914832f21e213339ed74f1",
+	"checkmark_square.fdf":          "d0ac69dac7a933e440a5005b1712edeb",
+	"checkmark_star.fdf":            "1326f152fb8158dffc08e5bb51cba1bc",
+	"test_fail.fdf":                 "9a90cef679d6b4c13017c73c2528ca75",
+}
+
+// Test filling (fdf merge) and flattening form data and annotations.
+func TestFdfMerging(t *testing.T) {
+	if len(fdfMergeCorpusFolder) == 0 {
+		if forceTest {
+			t.Fatalf("UNIDOC_FDFMERGE_TESTDATA not set")
+		}
+		t.Skipf("UNIDOC_FDFMERGE_TESTDATA not set")
+	}
+
+	files, err := ioutil.ReadDir(fdfMergeCorpusFolder)
+	if err != nil {
+		if forceTest {
+			t.Fatalf("Error opening %s: %v", fdfMergeCorpusFolder, err)
+		}
+		t.Skipf("Skipping flatten bench - unable to open UNIDOC_FDFMERGE_TESTDATA (%s)", fdfMergeCorpusFolder)
+	}
+
+	// Make a temporary folder and clean up after.
+	tempdir, err := ioutil.TempDir("", "unidoc_fdfmerge")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempdir)
+
+	matchcount := 0
+	for _, file := range files {
+		if strings.ToLower(filepath.Ext(file.Name())) != ".fdf" {
+			continue
+		}
+		fdfPath := filepath.Join(fdfMergeCorpusFolder, file.Name())
+		bareName := strings.TrimSuffix(file.Name(), ".fdf")
+		pdfPath := filepath.Join(fdfMergeCorpusFolder, bareName+".pdf")
+
+		// Ensure memory is garbage collected prior to running for consistency.
+		debug.FreeOSMemory()
+
+		t.Logf("%s", file.Name())
+		params := fdfMergeParams{
+			templatePath: pdfPath,
+			fdfPath:      fdfPath,
+			outPath:      filepath.Join(tempdir, "filled_flatten_1_"+bareName+".pdf"),
+			gsValidation: len(ghostscriptBinPath) > 0,
+		}
+		fdfMergeSingle(t, params)
+
+		hash, err := hashFile(params.outPath)
+		require.NoError(t, err)
+
+		knownHash, has := fdfMergeHashes[file.Name()]
+		if has {
+			require.Equal(t, knownHash, hash)
+			matchcount++
+		} else {
+			t.Logf("Output: %s", params.outPath)
+			t.Logf("%s - hash: %s not in the list of known hashes", file.Name(), hash)
+		}
+	}
+
+	// Ensure all the defined hashes were found.
+	require.Equal(t, len(fdfMergeHashes), matchcount)
+
+	t.Logf("FDF merge benchmark complete for %d cases in %s", matchcount, fdfMergeCorpusFolder)
+}
+
+type fdfMergeParams struct {
+	templatePath string // template PDF file.
+	fdfPath      string // form data FDF file.
+	outPath      string
+	gsValidation bool
+}
+
+func fdfMergeSingle(t *testing.T, params fdfMergeParams) {
+	measure := startMemoryMeasurement()
+
+	fdfData, err := fdf.LoadFromPath(params.fdfPath)
+	require.NoError(t, err)
+
+	f, err := os.Open(params.templatePath)
+	require.NoError(t, err)
+	defer f.Close()
+
+	pdfReader, err := model.NewPdfReader(f)
+	require.NoError(t, err)
+
+	// Populate the form data.
+	err = pdfReader.AcroForm.Fill(fdfData)
+	require.NoError(t, err)
+
+	// Flatten form.
+	fieldAppearance := annotator.FieldAppearance{OnlyIfMissing: true, RegenerateTextFields: true}
+
+	// NOTE: To customize certain styles try:
+	style := fieldAppearance.Style()
+	style.CheckmarkRune = '✖'
+	style.AutoFontSizeFraction = 0.70
+	fieldAppearance.SetStyle(style)
+
+	err = pdfReader.FlattenFields(true, fieldAppearance)
+	require.NoError(t, err)
+
+	// Write out.
+	model.SetPdfProducer("UniDoc")
+	pdfWriter := model.NewPdfWriter()
+	pdfWriter.SetForms(nil)
+
+	for _, p := range pdfReader.PageList {
+		err = pdfWriter.AddPage(p)
+		require.NoError(t, err)
+	}
+
+	fout, err := os.Create(params.outPath)
+	require.NoError(t, err)
+	defer fout.Close()
+
+	err = pdfWriter.Write(fout)
+	require.NoError(t, err)
+
+	measure.Stop()
+	summary := measure.Summary()
+	t.Logf("%s - summary %s", params.templatePath, summary)
+}

internal/e2etest/fdfmerge_test.go

@@ -681,6 +681,15 @@ func (r *PdfReader) newPdfFieldFromIndirectObject(container *core.PdfIndirectObj
 		for _, obj := range kids.Elements() {
 			container, isIndirect := core.GetIndirect(obj)
 			if !isIndirect {
+				stream, ok := core.GetStream(obj)
+				if ok && stream.PdfObjectDictionary != nil {
+					nodeType, ok := core.GetNameVal(stream.Get("Type"))
+					if ok && nodeType == "Metadata" {
+						common.Log.Debug("ERROR: form field Kids array contains invalid Metadata stream. Skipping.")
+						continue
+					}
+				}
+
 				return nil, errors.New("not an indirect object (form field)")
 			}