Use hash to keep TensorFlow model up to date

.dockerignore

@@ -5,6 +5,7 @@
 /frontend/node_modules/*
 /assets/server/public/build/*
 /assets/testdata
+/assets/backups
 /assets/tensorflow
 Dockerfile
 /photoprism

.gitignore

@@ -6,6 +6,7 @@
 /frontend/node_modules/*
 /frontend/tests/result.html
 /assets/testdata
+/assets/backups
 /assets/tensorflow
 *.log
 

.travis.yml

@@ -21,7 +21,7 @@ after_script:
 
 deploy:
   provider: script
-  script: make deploy-photoprism deploy-demo
+  script: make docker-photoprism docker-demo
   skip_cleanup: true
   on:
-    branch: master
+    branch: master

Dockerfile

@@ -1,4 +1,4 @@
-FROM photoprism/development:20190501
+FROM photoprism/development:20190503
 
 # Set up project directory
 WORKDIR "/go/src/github.com/photoprism/photoprism"

Makefile

@@ -42,24 +42,26 @@ test-coverage:
 	go tool cover -html=coverage.txt -o coverage.html
 clean:
 	rm -f $(BINARY_NAME)
+	rm -rf assets/testdata
+	rm -rf assets/backups
 download:
-	scripts/download-assets.sh
-deploy-photoprism:
-	scripts/docker-build.sh photoprism $(DOCKER_TAG)
-	scripts/docker-push.sh photoprism $(DOCKER_TAG)
-deploy-demo:
-	scripts/docker-build.sh demo $(DOCKER_TAG)
-	scripts/docker-push.sh demo $(DOCKER_TAG)
-deploy-development:
+	scripts/download-nasnet.sh
+docker-development:
 	scripts/docker-build.sh development $(DOCKER_TAG)
 	scripts/docker-push.sh development $(DOCKER_TAG)
-deploy-tensorflow:
+docker-photoprism:
+	scripts/docker-build.sh photoprism $(DOCKER_TAG)
+	scripts/docker-push.sh photoprism $(DOCKER_TAG)
+docker-demo:
+	scripts/docker-build.sh demo $(DOCKER_TAG)
+	scripts/docker-push.sh demo $(DOCKER_TAG)
+docker-tensorflow:
 	scripts/docker-build.sh tensorflow $(DOCKER_TAG)
 	scripts/docker-push.sh tensorflow $(DOCKER_TAG)
-deploy-darktable:
+docker-darktable:
 	scripts/docker-build.sh darktable $(DARKTABLE_VERSION)
 	scripts/docker-push.sh darktable $(DARKTABLE_VERSION)
-deploy-tidb:
+docker-tidb:
 	scripts/docker-build.sh tidb $(TIDB_VERSION)
 	scripts/docker-push.sh tidb $(TIDB_VERSION)
 fmt:

docker/development/Dockerfile

@@ -84,10 +84,12 @@ ENV PATH $GOBIN:/usr/local/go/bin:$PATH
 ENV GO111MODULE on
 RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 
+ENV BUILD_DATE `date -u +%Y%m%d`
+
 # Download TensorFlow model and test files
 RUN rm -rf /tmp/* && mkdir -p /tmp/photoprism
-RUN wget "https://dl.photoprism.org/tensorflow/nasnet-20190501.zip" -O /tmp/photoprism/nasnet.zip
-RUN wget "https://dl.photoprism.org/fixtures/testdata-20190501.zip" -O /tmp/photoprism/testdata.zip
+RUN wget "https://dl.photoprism.org/tensorflow/nasnet.zip?$BUILD_DATE" -O /tmp/photoprism/nasnet.zip
+RUN wget "https://dl.photoprism.org/fixtures/testdata.zip?$BUILD_DATE" -O /tmp/photoprism/testdata.zip
 
 # Install goimports
 RUN env GO111MODULE=off /usr/local/go/bin/go get golang.org/x/tools/cmd/goimports

docker/photoprism/Dockerfile

@@ -1,4 +1,4 @@
-FROM photoprism/development:20190501 as build
+FROM photoprism/development:20190503 as build
 
 # Set up project directory
 WORKDIR "/go/src/github.com/photoprism/photoprism"

docker/tensorflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM photoprism/development:20190418
+FROM photoprism/development:20190503
 
 # Install Python and TensorFlow
 RUN apt-get update && apt-get install -y --no-install-recommends \

internal/context/test_context.go

@@ -16,7 +16,7 @@ import (
 
 const (
 	TestDataZip  = "/tmp/photoprism/testdata.zip"
-	TestDataURL  = "https://dl.photoprism.org/fixtures/testdata-20190501.zip"
+	TestDataURL  = "https://dl.photoprism.org/fixtures/testdata.zip"
 	TestDataHash = "1a59b358b80221ab3e76efb683ad72402f0b0844"
 )
 

scripts/download-assets.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-FILENAME="/tmp/photoprism/nasnet.zip"
-
-if [[ ! -e assets/tensorflow/nasnet/saved_model.pb ]]; then
-  if [[ ! -e ${FILENAME} ]]; then
-      mkdir -p /tmp/photoprism
-      wget "https://dl.photoprism.org/tensorflow/nasnet.zip" -O ${FILENAME}
-  fi
-
-  mkdir -p assets/tensorflow
-  unzip ${FILENAME} -d assets/tensorflow
-else
-  echo "TensorFlow model already downloaded."
-fi

scripts/download-nasnet.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+TODAY=`date -u +%Y%m%d`
+
+MODEL_NAME="NASNet Mobile"
+MODEL_URL="https://dl.photoprism.org/tensorflow/nasnet.zip?$TODAY"
+MODEL_PATH="assets/tensorflow/nasnet"
+MODEL_ZIP="/tmp/photoprism/nasnet.zip"
+MODEL_HASH="6a9450f89afa56b4539c0d7188f108f083c10fc9  $MODEL_ZIP"
+MODEL_VERSION="$MODEL_PATH/version.txt"
+MODEL_BACKUP="assets/backups/nasnet-$TODAY"
+
+echo "Installing $MODEL_NAME for TensorFlow..."
+
+# Check for update
+if [[ -f ${MODEL_ZIP} ]] && [[ `sha1sum ${MODEL_ZIP}` == ${MODEL_HASH} ]]; then
+  echo "Already up to date."
+  exit
+fi
+
+# Create directories
+mkdir -p /tmp/photoprism
+mkdir -p assets/tensorflow
+mkdir -p assets/backups
+
+# Download model
+echo "Downloading latest model from $MODEL_URL..."
+wget ${MODEL_URL} -O ${MODEL_ZIP}
+
+TMP_HASH=`sha1sum ${MODEL_ZIP}`
+
+echo ${TMP_HASH}
+
+# Create backup
+if [[ -e ${MODEL_PATH} ]]; then
+  echo "Creating backup of existing directory: $MODEL_BACKUP"
+  rm -rf ${MODEL_BACKUP}
+  mv ${MODEL_PATH} ${MODEL_BACKUP}
+fi
+
+# Unzip model
+unzip ${MODEL_ZIP} -d assets/tensorflow
+echo "$MODEL_NAME $TODAY $MODEL_HASH" > ${MODEL_VERSION}
+
+echo "Latest $MODEL_NAME installed."