Switch to a more comprehensive mimetype detection library (#231)

This commit is contained in:
Infinoid 2020-08-03 01:16:47 -04:00 committed by GitHub
parent 6ce2bd6b9f
commit 5eb6f32ff0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 56 additions and 21 deletions

2
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5
github.com/dustin/go-humanize v1.0.0 github.com/dustin/go-humanize v1.0.0
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
github.com/gabriel-vasile/mimetype v1.1.1
github.com/microcosm-cc/bluemonday v1.0.2 github.com/microcosm-cc/bluemonday v1.0.2
github.com/minio/sha256-simd v0.1.1 github.com/minio/sha256-simd v0.1.1
github.com/russross/blackfriday v1.5.1 github.com/russross/blackfriday v1.5.1
@ -15,5 +16,4 @@ require (
github.com/zeebo/bencode v1.0.0 github.com/zeebo/bencode v1.0.0
github.com/zenazn/goji v0.9.0 github.com/zenazn/goji v0.9.0
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
gopkg.in/h2non/filetype.v1 v1.0.5
) )

4
go.sum
View File

@ -15,6 +15,8 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc=
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
github.com/gabriel-vasile/mimetype v1.1.1 h1:qbN9MPuRf3bstHu9zkI9jDWNfH//9+9kHxr9oRBBBOA=
github.com/gabriel-vasile/mimetype v1.1.1/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@ -68,8 +70,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y=
gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

View File

@ -7,8 +7,8 @@ import (
"unicode" "unicode"
"github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends"
"github.com/gabriel-vasile/mimetype"
"github.com/minio/sha256-simd" "github.com/minio/sha256-simd"
"gopkg.in/h2non/filetype.v1"
) )
func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) {
@ -21,7 +21,7 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) {
// Get first 512 bytes for mimetype detection // Get first 512 bytes for mimetype detection
header := make([]byte, 512) header := make([]byte, 512)
_, err = teeReader.Read(header) headerlen, err := teeReader.Read(header)
if err != nil { if err != nil {
return return
} }
@ -47,17 +47,8 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) {
// Use the bytes we extracted earlier and attempt to determine the file // Use the bytes we extracted earlier and attempt to determine the file
// type // type
kind, err := filetype.Match(header) kind := mimetype.Detect(header[:headerlen])
if err != nil { m.Mimetype = kind.String()
m.Mimetype = "application/octet-stream"
return m, err
} else if kind.MIME.Value != "" {
m.Mimetype = kind.MIME.Value
} else if printable(header) {
m.Mimetype = "text/plain"
} else {
m.Mimetype = "application/octet-stream"
}
return return
} }

View File

@ -1,8 +1,10 @@
package helpers package helpers
import ( import (
"bytes"
"strings" "strings"
"testing" "testing"
"unicode/utf16"
) )
func TestGenerateMetadata(t *testing.T) { func TestGenerateMetadata(t *testing.T) {
@ -17,7 +19,7 @@ func TestGenerateMetadata(t *testing.T) {
t.Fatalf("Sha256sum was %q instead of expected value of %q", m.Sha256sum, expectedSha256sum) t.Fatalf("Sha256sum was %q instead of expected value of %q", m.Sha256sum, expectedSha256sum)
} }
expectedMimetype := "text/plain" expectedMimetype := "text/plain; charset=utf-8"
if m.Mimetype != expectedMimetype { if m.Mimetype != expectedMimetype {
t.Fatalf("Mimetype was %q instead of expected value of %q", m.Mimetype, expectedMimetype) t.Fatalf("Mimetype was %q instead of expected value of %q", m.Mimetype, expectedMimetype)
} }
@ -27,3 +29,45 @@ func TestGenerateMetadata(t *testing.T) {
t.Fatalf("Size was %d instead of expected value of %d", m.Size, expectedSize) t.Fatalf("Size was %d instead of expected value of %d", m.Size, expectedSize)
} }
} }
func TestTextCharsets(t *testing.T) {
// verify that different text encodings are detected and passed through
orig := "This is a text string"
utf16 := utf16.Encode([]rune(orig))
utf16LE := make([]byte, len(utf16)*2+2)
utf16BE := make([]byte, len(utf16)*2+2)
utf8 := []byte(orig)
utf16LE[0] = 0xff
utf16LE[1] = 0xfe
utf16BE[0] = 0xfe
utf16BE[1] = 0xff
for i := 0; i < len(utf16); i++ {
lsb := utf16[i] & 0xff
msb := utf16[i] >> 8
utf16LE[i*2+2] = byte(lsb)
utf16LE[i*2+3] = byte(msb)
utf16BE[i*2+2] = byte(msb)
utf16BE[i*2+3] = byte(lsb)
}
testcases := []struct {
data []byte
extension string
mimetype string
}{
{mimetype: "text/plain; charset=utf-8", data: utf8},
{mimetype: "text/plain; charset=utf-16le", data: utf16LE},
{mimetype: "text/plain; charset=utf-16be", data: utf16BE},
}
for i, testcase := range testcases {
r := bytes.NewReader(testcase.data)
m, err := GenerateMetadata(r)
if err != nil {
t.Fatalf("[%d] unexpected error return %v\n", i, err)
}
if m.Mimetype != testcase.mimetype {
t.Errorf("[%d] Expected mimetype '%s', got mimetype '%s'\n", i, testcase.mimetype, m.Mimetype)
}
}
}

View File

@ -18,8 +18,8 @@ import (
"github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry" "github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"github.com/gabriel-vasile/mimetype"
"github.com/zenazn/goji/web" "github.com/zenazn/goji/web"
"gopkg.in/h2non/filetype.v1"
) )
var FileTooLargeError = errors.New("File too large.") var FileTooLargeError = errors.New("File too large.")
@ -263,11 +263,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
header = header[:n] header = header[:n]
// Determine the type of file from header // Determine the type of file from header
kind, err := filetype.Match(header) kind := mimetype.Detect(header)
if err != nil || kind.Extension == "unknown" { if len(kind.Extension()) < 2 {
extension = "file" extension = "file"
} else { } else {
extension = kind.Extension extension = kind.Extension()[1:] // remove leading "."
} }
} }