beenull
/
linx-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add linx-cleanup tool 

This doesn't completely fix #116, but it makes setting up a cron job to
do cleanup much more pleasant.
This commit is contained in:
mutantmonkey 2017-05-01 21:25:56 -07:00
parent d8568e141f
commit b7fadd9676
15 changed files with 216 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -29,3 +29,4 @@ _testmain.go
linx-server
files/
meta/
linx-cleanup

15
backends/localfs/localfs.go
View File

@ -65,6 +65,21 @@ func (b LocalfsBackend) Size(key string) (int64, error) {
return fileInfo.Size(), nil
}
func (b LocalfsBackend) List() ([]string, error) {
var output []string
files, err := ioutil.ReadDir(b.basePath)
if err != nil {
return nil, err
}
for _, file := range files {
output = append(output, file.Name())
}
return output, nil
}
func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath}
}

23
backends/meta.go Normal file
View File

@ -0,0 +1,23 @@
package backends
import (
"errors"
"time"
)
type MetaBackend interface {
Get(key string) (Metadata, error)
Put(key string, metadata *Metadata) error
}
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
ShortURL string
}
var BadMetadata = errors.New("Corrupted metadata.")

73
backends/metajson/metajson.go Normal file
View File

@ -0,0 +1,73 @@
package metajson
import (
"bytes"
"encoding/json"
"time"
"github.com/andreimarcu/linx-server/backends"
)
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
ShortURL string `json:"short_url"`
}
type MetaJSONBackend struct {
storage backends.MetaStorageBackend
}
func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
mjson.ShortURL = metadata.ShortURL
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
}
func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) {
b, err := m.storage.Get(key)
if err != nil {
return metadata, backends.BadMetadata
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, backends.BadMetadata
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
metadata.ShortURL = mjson.ShortURL
return
}
func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend {
return MetaJSONBackend{storage: storage}
}

5
backends/backends.go → backends/storage.go
View File

@ -21,3 +21,8 @@ type StorageBackend interface {
ServeFile(key string, w http.ResponseWriter, r *http.Request)
Size(key string) (int64, error)
}
type MetaStorageBackend interface {
StorageBackend
List() ([]string, error)
}

2
delete.go
View File

@ -28,7 +28,7 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if metadata.DeleteKey == requestKey {
fileDelErr := fileBackend.Delete(filename)
metaDelErr := metaBackend.Delete(filename)
metaDelErr := metaStorageBackend.Delete(filename)
if (fileDelErr != nil) || (metaDelErr != nil) {
oopsHandler(c, w, r, RespPLAIN, "Could not delete")

3
display.go
View File

@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize"
"github.com/flosch/pongo2"
"github.com/microcosm-cc/bluemonday"
@ -32,7 +33,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
return
}
var expiryHuman string
if metadata.Expiry != neverExpire {
if metadata.Expiry != expiry.NeverExpire {
expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "")
}
sizeHuman := humanize.Bytes(uint64(metadata.Size))

23
expiry.go
View File

@ -3,6 +3,7 @@ package main
import (
"time"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize"
)
@ -21,14 +22,6 @@ type ExpirationTime struct {
Human string
}
var neverExpire = time.Unix(0, 0)
// Determine if a file with expiry set to "ts" has expired yet
func isTsExpired(ts time.Time) bool {
now := time.Now()
return ts != neverExpire && now.After(ts)
}
// Determine if the given filename is expired
func isFileExpired(filename string) (bool, error) {
metadata, err := metadataRead(filename)
@ -36,7 +29,7 @@ func isFileExpired(filename string) (bool, error) {
return false, err
}
return isTsExpired(metadata.Expiry), nil
return expiry.IsTsExpired(metadata.Expiry), nil
}
// Return a list of expiration times and their humanized versions
@ -45,16 +38,16 @@ func listExpirationTimes() []ExpirationTime {
actualExpiryInList := false
var expiryList []ExpirationTime
for _, expiry := range defaultExpiryList {
if Config.maxExpiry == 0 || expiry <= Config.maxExpiry {
if expiry == Config.maxExpiry {
for _, expiryEntry := range defaultExpiryList {
if Config.maxExpiry == 0 || expiryEntry <= Config.maxExpiry {
if expiryEntry == Config.maxExpiry {
actualExpiryInList = true
}
duration := time.Duration(expiry) * time.Second
duration := time.Duration(expiryEntry) * time.Second
expiryList = append(expiryList, ExpirationTime{
expiry,
humanize.RelTime(epoch, epoch.Add(duration), "", ""),
Seconds: expiryEntry,
Human: humanize.RelTime(epoch, epoch.Add(duration), "", ""),
})
}
}

13
expiry/expiry.go Normal file
View File

@ -0,0 +1,13 @@
package expiry
import (
"time"
)
var NeverExpire = time.Unix(0, 0)
// Determine if a file with expiry set to "ts" has expired yet
func IsTsExpired(ts time.Time) bool {
now := time.Now()
return ts != NeverExpire && now.After(ts)
}

5
fileserve.go
View File

@ -5,6 +5,7 @@ import (
"net/url"
"strings"
"github.com/andreimarcu/linx-server/backends"
"github.com/zenazn/goji/web"
)
@ -15,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == BadMetadata {
} else if err == backends.BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}
@ -72,7 +73,7 @@ func checkFile(filename string) error {
if expired {
fileBackend.Delete(filename)
metaBackend.Delete(filename)
metaStorageBackend.Delete(filename)
return NotFoundErr
}

45
linx-cleanup/cleanup.go Normal file
View File

@ -0,0 +1,45 @@
package main
import (
"flag"
"log"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/andreimarcu/linx-server/expiry"
)
func main() {
var filesDir string
var metaDir string
var noLogs bool
flag.StringVar(&filesDir, "filespath", "files/",
"path to files directory")
flag.StringVar(&metaDir, "metapath", "meta/",
"path to metadata directory")
flag.BoolVar(&noLogs, "nologs", false,
"don't log deleted files")
metaStorageBackend := localfs.NewLocalfsBackend(metaDir)
metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend := localfs.NewLocalfsBackend(filesDir)
files, err := metaStorageBackend.List()
if err != nil {
panic(err)
}
for _, filename := range files {
metadata, err := metaBackend.Get(filename)
if err != nil {
log.Printf("Failed to find metadata for %s", filename)
}
if expiry.IsTsExpired(metadata.Expiry) {
log.Printf("Delete %s", filename)
fileBackend.Delete(filename)
metaStorageBackend.Delete(filename)
}
}
}

75
meta.go
View File

@ -3,46 +3,25 @@ package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"sort"
"time"
"unicode"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri"
"gopkg.in/h2non/filetype.v1"
)
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
ShortURL string `json:"short_url"`
}
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
ShortURL string
}
var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) {
file, err := fileBackend.Open(fName)
if err != nil {
return
@ -145,59 +124,23 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
return
}
func metadataWrite(filename string, metadata *Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
mjson.ShortURL = metadata.ShortURL
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
func metadataWrite(filename string, metadata *backends.Metadata) error {
return metaBackend.Put(filename, metadata)
}
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := metaBackend.Get(filename)
func metadataRead(filename string) (metadata backends.Metadata, err error) {
metadata, err = metaBackend.Get(filename)
if err != nil {
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
newMData, err := generateMetadata(filename, expiry.NeverExpire, "")
if err != nil {
return metadata, err
}
metadataWrite(filename, &newMData)
b, err = metaBackend.Get(filename)
if err != nil {
return metadata, BadMetadata
}
metadata, err = metaBackend.Get(filename)
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, BadMetadata
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
metadata.ShortURL = mjson.ShortURL
return
}

7
server.go
View File

@ -16,6 +16,7 @@ import (
"github.com/GeertJohan/go.rice"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful"
@ -65,7 +66,8 @@ var staticBox *rice.Box
var timeStarted time.Time
var timeStartedStr string
var remoteAuthKeys []string
var metaBackend backends.StorageBackend
var metaStorageBackend backends.MetaStorageBackend
var metaBackend backends.MetaBackend
var fileBackend backends.StorageBackend
func setup() *web.Mux {
@ -124,7 +126,8 @@ func setup() *web.Mux {
Config.sitePath = "/"
}
metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
// Template setup

3
torrent.go
View File

@ -8,6 +8,7 @@ import (
"net/http"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/zeebo/bencode"
"github.com/zenazn/goji/web"
)
@ -74,7 +75,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == BadMetadata {
} else if err == backends.BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}

20
upload.go
View File

@ -15,6 +15,8 @@ import (
"strings"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri"
"github.com/zenazn/goji/web"
"gopkg.in/h2non/filetype.v1"
@ -41,7 +43,7 @@ type UploadRequest struct {
// Metadata associated with a file as it would actually be stored
type Upload struct {
Filename string // Final filename on disk
Metadata Metadata
Metadata backends.Metadata
}
func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -258,11 +260,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
}
// Get the rest of the metadata needed for storage
var expiry time.Time
var fileExpiry time.Time
if upReq.expiry == 0 {
expiry = neverExpire
fileExpiry = expiry.NeverExpire
} else {
expiry = time.Now().Add(upReq.expiry)
fileExpiry = time.Now().Add(upReq.expiry)
}
bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
@ -273,7 +275,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
return upload, errors.New("File too large")
}
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey)
if err != nil {
fileBackend.Delete(upload.Filename)
return
@ -342,14 +344,14 @@ func parseExpiry(expStr string) time.Duration {
if expStr == "" {
return time.Duration(Config.maxExpiry) * time.Second
} else {
expiry, err := strconv.ParseUint(expStr, 10, 64)
fileExpiry, err := strconv.ParseUint(expStr, 10, 64)
if err != nil {
return time.Duration(Config.maxExpiry) * time.Second
} else {
if Config.maxExpiry > 0 && (expiry > Config.maxExpiry || expiry == 0) {
expiry = Config.maxExpiry
if Config.maxExpiry > 0 && (fileExpiry > Config.maxExpiry || fileExpiry == 0) {
fileExpiry = Config.maxExpiry
}
return time.Duration(expiry) * time.Second
return time.Duration(fileExpiry) * time.Second
}
}
}