Update documentation

.gitignore

@@ -0,0 +1 @@
+dist/

.goreleaser.yaml

@@ -0,0 +1,35 @@
+before:
+  hooks:
+    # clean up/install modules
+    - go mod tidy
+builds:
+  - main: 
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+archives:
+  - replacements:
+      darwin: Darwin
+      linux: Linux
+      windows: Windows
+      386: i386
+      amd64: x86_64
+    format_overrides:
+      - goos: windows
+        format: zip
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^Merge:'
+      - '^docs:'
+      - '^test:'
+      - '^[Bb]ump'
+      - '^[Cc]lean'

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "cSpell.words": [
+        "outputter"
+    ]
+}

README.md

@@ -1,10 +1,10 @@
 # slicerdicer
 
 Slice and dice an image, turning it into many equal sized tiles. Useful
-for things like leaflet.js, with the Leaflet.Zoomify plugin.
+for tools like leaflet.js, to create interactive "slippy" maps.
 
-The image is sliced up into equal sized tiles, based on the command line 
+The image is sliced up into equal sized tiles, based on the command line
-option `--tile-size` (default 512 pixels). 
+option `--tile-size` (default 256 pixels).
 
 Once the tiling is finished, the original is resized to half its current
 dimensions (the orignal file on disk is not touched) and the process repeats.
@@ -19,14 +19,55 @@ the top left tile.
 
 ## Usage
 
-    slicerdicer --help
+    slicerdicer -help
 
-    slicerdicer --filename foo.png --tile-size 256 --concurrency 5
+    slicerdicer -filename large_image.png -tile-size 256 -concurrency 5
+
+## Output filenames
+
+The destination for the tiles can be changed with the `-basename` and
+`-path-template` options. The path template must contain 5 placeholders,
+in the following order:
+
+* `%s` basename, as per `-basename` (default `tile`)
+* `%d` zoom level
+* `%d` 'x' coordinate
+* `%d` 'y' coordinate
+* `%d` file format (jpg or png)
+
+The default template is `%s-%d-%d-%d.%s` which results in a flat structure
+with all files in the current directory.
+
+For example, using `-basename map` and
+`-path-template '%s/zoom-%d/%d-%d.%s'` will result in a file structure like:
+
+    map
+    ├── 0
+    │   └── 0-0.png
+    ├── 1
+    │   ├── 0-0.png
+    │   └── 1-0.png
+    └── 2
+        ├── 0-0.png
+        ├── 0-1.png
+        ├── 1-0.png
+        ├── 1-1.png
+        ├── 2-0.png
+        ├── 2-1.png
+        ├── 3-0.png
+        └── 3-1.png
+
+All tiles in a directory called `map`, with a second level directory for zoom
+level, each file named `x-y.png` within that.
 
 ## Notes
 
-It's going to eat some memory. 
+### Memory
 
 In my tests on an 32641 x 16471, 8-bit/color RGB PNG, memory usage peaks at
 around 2.7GB.
 
+### Speed
+
+On that same test image, the run takes around 63 seconds to create the 11179
+tiles, on my fairly underwhelming MacBookPro12,1 (dual core i5).

build-release.pl

@@ -1,37 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-open my $fh, "<", "main.go" || die $!;
-
-my $version;
-while (<$fh>) {
-  $version = $1 if /^const\s+currentVersion.*?"([\d\.]+)"/;
-}
-close $fh;
-
-die "no version?" unless defined $version;
-
-# so lazy
-system "rm", "-rf", "release", "dist";
-system "mkdir", "release";
-system "mkdir", "dist";
-
-my %build = (
-  win   => { env => { GOOS => 'windows', GOARCH => '386' }, filename => 'slicerdicer.exe' },
-  linux => { env => { GOOS => 'linux',   GOARCH => '386' }, filename => 'slicerdicer' },
-  mac   => { env => { GOOS => 'darwin',  GOARCH => '386' }, filename => 'slicerdicer' },
-); 
-
-foreach my $type (keys %build) {
-  mkdir "release/$type";
-}
-
-foreach my $type (keys %build) {
-  local $ENV{GOOS}   = $build{$type}->{env}->{GOOS};
-  local $ENV{GOARCH} = $build{$type}->{env}->{GOARCH};
-  system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
-  system "zip", "-j", "dist/slicerdicer-$type-$version.zip", ( glob "release/$type/*" );
-}
-

go.mod

@@ -0,0 +1,16 @@
+module github.com/tardisx/slicerdicer
+
+go 1.22.1
+
+require github.com/disintegration/imaging v1.6.2
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+require (
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect
+)

go.sum

@@ -0,0 +1,15 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -1,109 +1,254 @@
-package main;
+package main
 
-import "image"
+import (
-import "image/png"
+	"flag"
-import "github.com/disintegration/imaging"
+	"fmt"
-import "runtime"
+	"image"
-import "flag"
+	"image/jpeg"
-import "fmt"
+	"image/png"
-import "os"
+	"io"
+	"log"
+	"math"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
 
-const currentVersion = "0.01"
+	"github.com/disintegration/imaging"
+)
+
+type resultPrinter struct {
+	batchTotal   int
+	count        int
+	lastFilename string
+	ch           chan string
+}
+
+func (rp *resultPrinter) Reset(batchSize int) {
+	rp.batchTotal = batchSize
+	rp.count = 0
+	rp.lastFilename = ""
+	fmt.Print("\033[s")
+}
+
+func (rp *resultPrinter) Run() {
+	for last := range rp.ch {
+		rp.count++
+		rp.lastFilename = last
+		fmt.Print("\033[u\033[K") // restore the cursor position and clear the line
+		fmt.Printf("processing %5d/%5d - %s", rp.count, rp.batchTotal, rp.lastFilename)
+	}
+}
+
+const defaultPathTemplate = "%s-%d-%d-%d.%s"
 
 func main() {
+	t0 := time.Now()
+	var filename, baseName, outputFormat, pathTemplate string
+	var tileSize, concurrency int
 
-  filenamePtr    := flag.String("filename", "", "filename to open")
+	flag.StringVar(&filename, "filename", "", "filename to open")
-  tileSizePtr    := flag.Int   ("tile-size", 256, "tile size, in pixels")
+	flag.IntVar(&tileSize, "tile-size", 256, "tile size, in pixels")
-  concurrencyPtr := flag.Int ("concurrency", 5, "how many tiles to generate concurrently (threads)")
+	flag.IntVar(&concurrency, "concurrency", 5, "how many tiles to generate concurrently (threads)")
+	flag.StringVar(&baseName, "basename", "tile", "base of the output files")
+	flag.StringVar(&outputFormat, "format", "png", "output format (jpg/png)")
+	flag.StringVar(&pathTemplate, "path-template", defaultPathTemplate, "template for output files - base, zoom, x, y, format")
 
-  flag.Parse()
+	flag.Parse()
 
-  if (*filenamePtr == "") {
+	if filename == "" {
-    fmt.Println("Error: You must specify a filename with --filename");
+		fmt.Println("Error: You must specify a filename with -filename")
-    return;
+		return
-  }
+	}
 
-  fmt.Println("opening file:", *filenamePtr)
+	if outputFormat != "jpg" && outputFormat != "png" {
-  src, err := imaging.Open(*filenamePtr)
+		fmt.Println("Error: -format must be jpg or png")
-  if err != nil {
+		return
-    fmt.Println("Error: Could not open file:", err)
+	}
-    return;
-  }
 
-  size := src.Bounds().Max
+	log.Println("opening file:", filename)
+	src, err := os.Open(filename)
+	if err != nil {
+		fmt.Println("Error: Could not open file:", err)
+		return
+	}
 
-  tile_size_x := *tileSizePtr
+	processImage(src, baseName, pathTemplate, outputFormat, tileSize, concurrency, diskOutput{})
-  tile_size_y := *tileSizePtr
+	log.Printf("done in %.2f", time.Since(t0).Seconds())
-
-  // work out maximum zoom
-  var max_zoom int
-  zoom_test_size_x := size.X
-  zoom_test_size_y := size.Y
-  for max_zoom = 0 ; ; max_zoom++ {
-    if zoom_test_size_x < tile_size_x && zoom_test_size_y < tile_size_y {
-      break
-    }
-    zoom_test_size_x = zoom_test_size_x >> 1
-    zoom_test_size_y = zoom_test_size_y >> 1
-  }
-
-  z := max_zoom
-  fmt.Println("maximum zoom level is", max_zoom)
-
-  concurrency := *concurrencyPtr
-  sem := make(chan bool, concurrency)
-
-  fmt.Println("starting tiling with concurrency of", concurrency)
-
-  // outer loop for zoom
-  for {
-    if (z == max_zoom) {
-      // do nothing
-    } else {
-      // halve image size
-      src = imaging.Resize(src, size.X/2, 0, imaging.NearestNeighbor)
-      runtime.GC()
-      // recalculate size
-      size = src.Bounds().Max
-    }
-
-    fmt.Print(fmt.Sprintf("zoom level: %d (%d x %d)\n", z, size.X, size.Y))
-
-    for y := 0 ; y <= (size.Y / tile_size_y) ; y++ {
-      for x := 0 ; x <= (size.X / tile_size_x) ; x++ {
-        sem <- true
-        go tile(src, z, x, y, tile_size_x, tile_size_y, sem)
-      }
-
-    }
-
-    z--
-    if (z < 0) {
-      break
-    }
-  }
-
-  // drain at the end of each zoom level
-  // since we are about to modify the source image
-  for i := 0; i < cap(sem); i++ {
-    sem <- true
-  }
-
-  fmt.Println("done")
 }
 
-func tile (src image.Image, z, x, y int, tile_size_x, tile_size_y int, sem chan bool) {
+func processImage(input io.Reader, basename string, pathTemplate string, format string, tileSize int, concurrency int, output outputter) {
-  defer func() { <-sem }()
-  output_filename := fmt.Sprintf("tile-%d-%d-%d.png", z, x, y)
-  cropped := imaging.Crop(src, image.Rect(tile_size_x*x, tile_size_y*y, tile_size_x*x+tile_size_x, tile_size_y*y+tile_size_y));
 
-  writer, _ := os.Create(output_filename)
+	src, err := imaging.Decode(input)
-  err := png.Encode(writer, cropped)
+	if err != nil {
-  if err != nil {
+		panic(err)
-    fmt.Println(err)
+	}
-  }
-  writer.Close()
 
-  runtime.GC()
+	size := src.Bounds().Max
-  return;
+
+	tile_size_x := tileSize
+	tile_size_y := tileSize
+
+	// work out maximum zoom
+	var max_zoom int
+	zoom_test_size_x := size.X
+	zoom_test_size_y := size.Y
+	for max_zoom = 0; ; max_zoom++ {
+		if zoom_test_size_x <= tile_size_x && zoom_test_size_y <= tile_size_y {
+			break
+		}
+		zoom_test_size_x = zoom_test_size_x >> 1
+		zoom_test_size_y = zoom_test_size_y >> 1
+	}
+
+	z := max_zoom
+	log.Println("maximum zoom level is", max_zoom)
+	log.Println("starting tiling with concurrency of", concurrency)
+	log.Printf("basename '%s'", basename)
+
+	results := make(chan string)
+	rp := resultPrinter{
+		batchTotal:   0,
+		count:        0,
+		lastFilename: "",
+		ch:           results,
+	}
+
+	// start the tileWorkers
+	jobs := make(chan tileJob)
+	for i := 0; i < concurrency; i++ {
+		go tileWorker(jobs, results)
+	}
+
+	go func() {
+		rp.Run()
+	}()
+
+	// outer loop for zoom
+	for {
+
+		if z == max_zoom {
+			// do nothing
+		} else {
+			// halve image size
+			log.Print("resizing for next zoom level")
+			src = imaging.Resize(src, size.X/2, 0, imaging.NearestNeighbor)
+			// recalculate size
+			size = src.Bounds().Max
+		}
+
+		log.Printf("zoom level: %d (%d x %d)\n", z, size.X, size.Y)
+
+		// size 257 => 2
+		// size 256 => 1
+		// size 255 => 1
+		yTiles := int(math.Ceil(float64(size.Y) / float64(tile_size_y)))
+		xTiles := int(math.Ceil(float64(size.X) / float64(tile_size_x)))
+
+		if z == 0 {
+			xTiles = 1
+			yTiles = 1
+		}
+		tilesToRender := xTiles * yTiles
+
+		rp.Reset(tilesToRender)
+
+		wg := sync.WaitGroup{}
+		wg.Add(tilesToRender)
+
+		log.Printf("rendering %d tiles", tilesToRender)
+
+		for y := 0; y < yTiles; y++ {
+			for x := 0; x < xTiles; x++ {
+				jobs <- tileJob{
+					baseName:     basename,
+					pathTemplate: pathTemplate,
+					format:       format,
+					src:          src,
+					zoom:         z,
+					x:            x,
+					y:            y,
+					tileSizeX:    tile_size_x,
+					tileSizeY:    tile_size_y,
+					wg:           &wg,
+					output:       output,
+				}
+			}
+
+		}
+		wg.Wait() // wait for all tiles to be generated for this zoom level
+		z--
+		if z < 0 {
+			break
+		}
+
+		// let the last progress be printed out
+		// yes I know this is ugly :-)
+		time.Sleep(time.Millisecond * 10)
+		fmt.Print("\033[u\033[K") // restore the cursor position and clear the line
+	}
+	close(results)
+	log.Printf("all tiles complete")
+}
+
+type outputter interface {
+	CreatePathAndFile(fn string) (io.WriteCloser, error)
+}
+
+type diskOutput struct {
+}
+
+func (do diskOutput) CreatePathAndFile(fn string) (io.WriteCloser, error) {
+	dir, _ := filepath.Split(fn)
+	if dir == "" {
+		// no need to do anything, going in current dir
+	} else {
+		err := os.MkdirAll(dir, 0777)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	writer, err := os.Create(fn)
+	return writer, err
+}
+
+type tileJob struct {
+	baseName     string
+	pathTemplate string
+	format       string
+	src          image.Image
+	zoom         int
+	x, y         int
+	tileSizeX    int
+	tileSizeY    int
+	output       outputter
+	wg           *sync.WaitGroup
+}
+
+func tileWorker(jobs <-chan tileJob, results chan<- string) {
+	for j := range jobs {
+		output_filename := fmt.Sprintf(j.pathTemplate, j.baseName, j.zoom, j.x, j.y, j.format)
+		cropped := imaging.Crop(j.src, image.Rect(j.tileSizeX*j.x, j.tileSizeY*j.y, j.tileSizeX*j.x+j.tileSizeX, j.tileSizeY*j.y+j.tileSizeY))
+
+		writer, err := j.output.CreatePathAndFile(output_filename)
+		if err != nil {
+			log.Fatalf("could not create path/file: '%s'", err.Error())
+		}
+		if j.format == "png" {
+			err = png.Encode(writer, cropped)
+		} else if j.format == "jpg" {
+			err = jpeg.Encode(writer, cropped, &jpeg.Options{
+				Quality: 40,
+			})
+		} else {
+			panic("bad format")
+		}
+		if err != nil {
+			panic(err)
+		}
+		writer.Close()
+		results <- output_filename
+		j.wg.Done()
+	}
 }

slice_test.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type testWriteCloser struct {
+	written int
+	closed  bool
+}
+
+func (twc *testWriteCloser) Close() error {
+	twc.closed = true
+	return nil
+}
+
+func (twc *testWriteCloser) Write(b []byte) (int, error) {
+	twc.written += len(b)
+	return len(b), nil
+}
+
+type testOutputter struct {
+	closers map[string]*testWriteCloser
+}
+
+func (to *testOutputter) CreatePathAndFile(fn string) (io.WriteCloser, error) {
+	to.closers[fn] = &testWriteCloser{}
+	return to.closers[fn], nil
+}
+
+func (to *testOutputter) Dump() string {
+	out := ""
+	for k, v := range to.closers {
+		out += fmt.Sprintf("%20s: %6d/%t\n", k, v.written, v.closed)
+	}
+	return out
+}
+
+func NewTestOutputter() *testOutputter {
+	to := testOutputter{}
+	to.closers = make(map[string]*testWriteCloser)
+	return &to
+}
+
+func TestXxx(t *testing.T) {
+
+	// testdata/gold.jpg: JPEG image data, JFIF standard 1.01, 794x447
+	f, err := os.Open("testdata/gold.jpg")
+	if err != nil {
+		panic(err)
+	}
+
+	// with a tile size larger than the image itself, we should
+	// get a single image at zoom level 0
+	testOutputter := NewTestOutputter()
+	processImage(f, "indy", defaultPathTemplate, "jpg", 1000, 1, testOutputter)
+	t.Log(testOutputter.Dump())
+	assert.Equal(t, 1, len(testOutputter.closers))
+	assert.Equal(t, 28082, testOutputter.closers["indy-0-0-0.jpg"].written)
+
+	f, _ = os.Open("testdata/gold.jpg")
+
+	// with a tilesize smaller than *one* of the dimensions, 3 tiles
+	// 1@zoom 0, 2@zoom 1
+	testOutputter = NewTestOutputter()
+	processImage(f, "indy", defaultPathTemplate, "jpg", 500, 1, testOutputter)
+	t.Log(testOutputter.Dump())
+	assert.Equal(t, 3, len(testOutputter.closers))
+	assert.Equal(t, 10304, testOutputter.closers["indy-0-0-0.jpg"].written)
+	assert.Equal(t, 18014, testOutputter.closers["indy-1-0-0.jpg"].written)
+	assert.Equal(t, 10749, testOutputter.closers["indy-1-1-0.jpg"].written)
+
+	f, _ = os.Open("testdata/gold.jpg")
+
+	// with a tilesize smaller than *both* of the dimensions, 5 tiles
+	// zoom 0, zoom 1 x 4
+	testOutputter = NewTestOutputter()
+	processImage(f, "indy", defaultPathTemplate, "jpg", 400, 1, testOutputter)
+	t.Log(testOutputter.Dump())
+	assert.Equal(t, 5, len(testOutputter.closers))
+	assert.Equal(t, 10304, testOutputter.closers["indy-0-0-0.jpg"].written)
+	assert.Equal(t, 12607, testOutputter.closers["indy-1-0-0.jpg"].written)
+	assert.Equal(t, 12050, testOutputter.closers["indy-1-1-0.jpg"].written)
+	assert.Equal(t, 2722, testOutputter.closers["indy-1-0-1.jpg"].written)
+	assert.Equal(t, 2477, testOutputter.closers["indy-1-1-1.jpg"].written)
+
+}