Merge pull request #165 from unidoc/release/v3.2.0

Release unipdf v3.2.0

Author: gunnsth

.travis.yml

@@ -0,0 +1,35 @@
+language: go
+sudo: required
+go:
+  - 1.12.x
+env:
+  global:
+    - CGO_ENABLED="0"
+    - GO111MODULE="on"
+    - TMPDIR="$(echo -n $HOME/tmp)"
+    - UNIDOC_GS_BIN_PATH="/usr/bin/gs"
+    - UNIDOC_EXTRACT_FORCETEST="0"
+    - UNIDOC_E2E_FORCE_TESTS="0"
+    - UNIDOC_EXTRACT_TESTDATA=""
+    - UNIDOC_RENDERTEST_BASELINE_PATH=""
+    - UNIDOC_PASSTHROUGH_TESTDATA=""
+    - UNIDOC_ALLOBJECTS_TESTDATA=""
+    - UNIDOC_SPLIT_TESTDATA=""
+    - UNIDOC_EXTRACT_IMAGES_TESTDATA=""
+    - UNIDOC_JBIG2_TESTDATA=""
+    - UNIDOC_FDFMERGE_TESTDATA=""
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install -y git ghostscript
+  - go get golang.org/x/lint/golint
+  - go get -t ./...
+  - mkdir -p $TMPDIR
+script:
+  - go vet ./...
+  - golint ./...
+  - go test -v ./...
+  - find $TMPDIR -maxdepth 1 -name "*.pdf" -print0 | xargs -t -n 1 -0 gs -dNOPAUSE -dBATCH -sDEVICE=nullpage -sPDFPassword=password -dPDFSTOPONERROR -dPDFSTOPONWARNING
+  - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=./... ./...
+  - ./.travis/build_examples.sh
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

.travis/build_examples.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+repo_path="$HOME/projects/unipdf-examples"
+git clone https://github.com/unidoc/unipdf-examples.git $repo_path
+cd $repo_path
+
+build_branch="$TRAVIS_BRANCH"
+if [[ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]]; then
+    build_branch="$TRAVIS_PULL_REQUEST_BRANCH"
+fi
+if [[ -z "$build_branch" ]]; then
+    build_branch="development"
+fi
+
+target_branch="development"
+if [[ `git ls-remote origin "$build_branch"` ]]; then
+    target_branch="$build_branch"
+fi
+
+git checkout $target_branch
+echo "replace github.com/unidoc/unipdf/v3 => $TRAVIS_BUILD_DIR" >> go.mod
+./build_examples.sh

Jenkinsfile

@@ -15,6 +15,7 @@ node {
     env.UNIDOC_PASSTHROUGH_TESTDATA="/home/jenkins/corpus/unidoc-e2e-testdata"
     env.UNIDOC_ALLOBJECTS_TESTDATA="/home/jenkins/corpus/unidoc-e2e-testdata"
     env.UNIDOC_SPLIT_TESTDATA="/home/jenkins/corpus/unidoc-e2e-split-testdata"
+    env.UNIDOC_EXTRACT_IMAGES_TESTDATA="/home/jenkins/corpus/unidoc-e2e-extract-images-testdata"
     env.UNIDOC_JBIG2_TESTDATA="/home/jenkins/corpus/jbig2-testdata"
     env.UNIDOC_FDFMERGE_TESTDATA="/home/jenkins/corpus/fdfmerge-testdata"
     env.UNIDOC_GS_BIN_PATH="/usr/bin/gs"
@@ -85,7 +86,7 @@ node {
             sh("printenv")
 
             // Pull unipdf-examples from connected branch, or master otherwise.
-            def examplesBranch = "v3"
+            def examplesBranch = "development"
 
             // Check if connected branch is defined explicitly.
             def safeName = env.BRANCH_NAME.replaceAll(/[\/\.]/, '')

common/version.go

@@ -11,12 +11,12 @@ import (
 )
 
 const releaseYear = 2019
-const releaseMonth = 8
-const releaseDay = 4
-const releaseHour = 11
-const releaseMin = 40
+const releaseMonth = 9
+const releaseDay = 7
+const releaseHour = 13
+const releaseMin = 10
 
 // Version holds version information, when bumping this make sure to bump the released at stamp also.
-const Version = "3.1.1"
+const Version = "3.2.0"
 
 var ReleasedAt = time.Date(releaseYear, releaseMonth, releaseDay, releaseHour, releaseMin, 0, 0, time.UTC)

core/encoding.go

@@ -2138,9 +2138,7 @@ func (enc *JBIG2Encoder) DecodeBytes(encoded []byte) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	// Inverse the data representation if the decoder is marked as 'isChocolateData'.
-	bm.InverseData(enc.IsChocolateData)
+	bm.GetVanillaData()
 
 	// By default the bitmap data contains the rowstride padding.
 	// In order to get rid of the rowstride padding use the bitmap.GetUnpaddedData method.

creator/creator.go

@@ -291,10 +291,11 @@ func (c *Creator) initContext() {
 }
 
 // NewPage adds a new Page to the Creator and sets as the active Page.
-func (c *Creator) NewPage() {
+func (c *Creator) NewPage() *model.PdfPage {
 	page := c.newPage()
 	c.pages = append(c.pages, page)
 	c.context.Page++
+	return page
 }
 
 // AddPage adds the specified page to the creator.
@@ -482,6 +483,8 @@ func (c *Creator) finalize() error {
 
 	for idx, page := range c.pages {
 		c.setActivePage(page)
+
+		// Draw page header.
 		if c.drawHeaderFunc != nil {
 			// Prepare a block to draw on.
 			// Header is drawn on the top of the page. Has width of the page, but height limited to
@@ -493,12 +496,14 @@ func (c *Creator) finalize() error {
 			}
 			c.drawHeaderFunc(headerBlock, args)
 			headerBlock.SetPos(0, 0)
-			err := c.Draw(headerBlock)
-			if err != nil {
+
+			if err := c.Draw(headerBlock); err != nil {
 				common.Log.Debug("ERROR: drawing header: %v", err)
 				return err
 			}
 		}
+
+		// Draw page footer.
 		if c.drawFooterFunc != nil {
 			// Prepare a block to draw on.
 			// Footer is drawn on the bottom of the page. Has width of the page, but height limited
@@ -510,25 +515,25 @@ func (c *Creator) finalize() error {
 			}
 			c.drawFooterFunc(footerBlock, args)
 			footerBlock.SetPos(0, c.pageHeight-footerBlock.height)
-			err := c.Draw(footerBlock)
-			if err != nil {
+
+			if err := c.Draw(footerBlock); err != nil {
 				common.Log.Debug("ERROR: drawing footer: %v", err)
 				return err
 			}
 		}
 
-		// Draw blocks to pages.
+		// Draw page blocks.
 		block, ok := c.pageBlocks[page]
 		if !ok {
-			return errors.New("could not find page block")
+			continue
 		}
 		if err := block.drawToPage(page); err != nil {
+			common.Log.Debug("ERROR: drawing page %d blocks: %v", idx+1, err)
 			return err
 		}
 	}
 
 	c.finalized = true
-
 	return nil
 }
 
@@ -600,7 +605,9 @@ func (c *Creator) Draw(d Drawable) error {
 // Write output of creator to io.Writer interface.
 func (c *Creator) Write(ws io.Writer) error {
 	if !c.finalized {
-		c.finalize()
+		if err := c.finalize(); err != nil {
+			return err
+		}
 	}
 
 	pdfWriter := model.NewPdfWriter()

creator/paragraph.go

@@ -7,7 +7,6 @@ package creator
 
 import (
 	"errors"
-	"fmt"
 	"strconv"
 
 	"github.com/unidoc/unipdf/v3/common"
@@ -489,7 +488,6 @@ func drawParagraphOnBlock(blk *Block, p *Paragraph, ctx DrawContext) (DrawContex
 		enc := p.textFont.Encoder()
 
 		var encoded []byte
-		isCID := p.textFont.IsCID()
 		for _, r := range runes {
 			if r == ' ' { // TODO: What about \t and other spaces.
 				if len(encoded) > 0 {
@@ -498,19 +496,11 @@ func drawParagraphOnBlock(blk *Block, p *Paragraph, ctx DrawContext) (DrawContex
 				}
 				objs = append(objs, core.MakeFloat(-spaceWidth))
 			} else {
-				code, ok := enc.RuneToCharcode(r)
-				if !ok {
-					err := fmt.Errorf("unsupported rune in text encoding: %#x (%c)", r, r)
-					common.Log.Debug("%s", err)
-					return ctx, err
-				}
-				// TODO(dennwc): this should not be done manually; encoder should do this
-				if isCID {
-					hi, lo := code>>8, code&0xff
-					encoded = append(encoded, byte(hi), byte(lo))
-				} else {
-					encoded = append(encoded, byte(code))
+				if _, ok := enc.RuneToCharcode(r); !ok {
+					common.Log.Debug("unsupported rune in text encoding: %#x (%c)", r, r)
+					continue
 				}
+				encoded = append(encoded, enc.Encode(string(r))...)
 			}
 		}
 		if len(encoded) > 0 {

creator/styled_paragraph.go

@@ -761,6 +761,10 @@ func drawStyledParagraphOnBlock(blk *Block, p *StyledParagraph, ctx DrawContext)
 
 					chunkWidths[k] += spaceWidth * fontSize
 				} else {
+					if _, ok := enc.RuneToCharcode(rn); !ok {
+						common.Log.Debug("unsupported rune in text encoding: %#x (%c)", rn, rn)
+						continue
+					}
 					encStr = append(encStr, enc.Encode(string(rn))...)
 				}
 			}

creator/text_chunk.go

@@ -44,10 +44,9 @@ func newExternalLinkAnnotation(url string) *model.PdfAnnotation {
 	annotation.BS = bs.ToPdfObject()
 
 	// Set link destination.
-	action := core.MakeDict()
-	action.Set(core.PdfObjectName("S"), core.MakeName("URI"))
-	action.Set(core.PdfObjectName("URI"), core.MakeString(url))
-	annotation.A = action
+	action := model.NewPdfActionURI()
+	action.URI = core.MakeString(url)
+	annotation.SetAction(action.PdfAction)
 
 	return annotation.PdfAnnotation
 }

extractor/text.go

@@ -1180,14 +1180,46 @@ var (
 // sortPosition sorts a text list by its elements' positions on a page.
 // Sorting is by orientation then top to bottom, left to right when page is orientated so that text
 // is horizontal.
+// Text is considered to be on different lines if the lines' orientedStart.Y differs by more than `tol`.
 func (pt *PageText) sortPosition(tol float64) {
+	if len(pt.marks) == 0 {
+		return
+	}
+
+	// For grouping data vertically into lines, it is necessary to have the data presorted by
+	// descending y position.
+	sort.SliceStable(pt.marks, func(i, j int) bool {
+		ti, tj := pt.marks[i], pt.marks[j]
+		if ti.orient != tj.orient {
+			return ti.orient < tj.orient
+		}
+		return ti.orientedStart.Y >= tj.orientedStart.Y
+	})
+
+	// Cluster the marks into y-clusters by relative y proximity. Each cluster is our guess of what
+	// makes up a line of text.
+	clusters := make([]int, len(pt.marks))
+	cluster := 0
+	clusters[0] = cluster
+	for i := 1; i < len(pt.marks); i++ {
+		if pt.marks[i-1].orient != pt.marks[i].orient {
+			cluster++
+		} else {
+			if pt.marks[i-1].orientedStart.Y - pt.marks[i].orientedStart.Y > tol {
+				cluster++
+			}
+		}
+		clusters[i] = cluster
+	}
+
+	// Sort by y-cluster and x.
 	sort.SliceStable(pt.marks, func(i, j int) bool {
 		ti, tj := pt.marks[i], pt.marks[j]
 		if ti.orient != tj.orient {
 			return ti.orient < tj.orient
 		}
-		if math.Abs(ti.orientedStart.Y-tj.orientedStart.Y) > tol {
-			return ti.orientedStart.Y > tj.orientedStart.Y
+		if clusters[i] != clusters[j] {
+			return clusters[i] < clusters[j]
 		}
 		return ti.orientedStart.X < tj.orientedStart.X
 	})