From c5c2de7233d86b42ab91a7cb70dd9facb003bb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Auger?= Date: Tue, 25 Aug 2020 16:25:44 +0200 Subject: [PATCH 1/3] Add file upload support --- go.mod | 3 ++ go.sum | 9 ++++ handlers.go | 108 +++++++++++++++++++++++++++++++++++++ internal/hub/hub.go | 1 + internal/hub/peer.go | 31 ++++++++++- internal/hub/room.go | 4 +- internal/upload/store.go | 102 +++++++++++++++++++++++++++++++++++ main.go | 68 +++++++++++++++++++++++ static/static/app.js | 50 ++++++++++++++++- static/static/client.js | 1 + static/static/style.css | 10 ++++ static/templates/room.html | 22 +++++++- 12 files changed, 402 insertions(+), 7 deletions(-) create mode 100644 internal/upload/store.go diff --git a/go.mod b/go.mod index 52a6e82..0093783 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,16 @@ module github.com/knadh/niltalk go 1.13 require ( + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/go-chi/chi v4.1.0+incompatible github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/websocket v1.4.2 + github.com/karrick/tparse/v2 v2.8.2 github.com/knadh/koanf v0.9.1 github.com/knadh/stuffbin v1.1.0 github.com/kr/pretty v0.1.0 // indirect github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 890d0a2..597a7b8 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -14,6 +16,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/karrick/tparse/v2 v2.8.2 h1:NhvrrB7nXYa0VLn0JKn9L3oG/GZN+LB/+g5QfWE30rU= +github.com/karrick/tparse/v2 v2.8.2/go.mod h1:OzmKMqNal7LYYHaO/Ie1f/wXmLWAaGKwJmxUFNQCVxg= github.com/knadh/koanf v0.9.1 h1:qfcwiF9/Z8buTJ0QXaZvOxJ6eKJmOiiWKP/PktiW5RE= github.com/knadh/koanf v0.9.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= github.com/knadh/stuffbin v1.1.0 h1:f5S5BHzZALjuJEgTIOMC9NidEnBJM7Ze6Lu1GHR/lwU= @@ -35,6 +39,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 h1:fpnn/HnJONpIu6hkXi1u/7rR0NzilgWr4T0JmWkEitk= golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -46,8 +52,11 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ix golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers.go b/handlers.go index f2f7cd4..92c74cb 100644 --- a/handlers.go +++ b/handlers.go @@ -4,13 +4,20 @@ import ( "context" "encoding/json" "errors" + "fmt" "io/ioutil" + "log" "net/http" + "strings" + "sync" + "time" "github.com/go-chi/chi" "github.com/gorilla/websocket" "github.com/knadh/niltalk/internal/hub" + "github.com/knadh/niltalk/internal/upload" "golang.org/x/crypto/bcrypt" + "golang.org/x/time/rate" ) const ( @@ -327,3 +334,104 @@ func readJSONReq(r *http.Request, o interface{}) error { } return json.Unmarshal(b, o) } + +// handleUpload handles file uploads. +func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Duration, rlCount float64, rlBurst int) func(w http.ResponseWriter, r *http.Request) { + + type roomLimiter struct { + limiter *rate.Limiter + expire time.Time + } + var mu sync.Mutex + roomLimiters := map[string]roomLimiter{} + go func() { + t := time.NewTicker(rlPeriod + (time.Minute)) + defer t.Stop() + for range t.C { + now := time.Now() + mu.Lock() + for k, r := range roomLimiters { + if r.expire.Before(now) { + delete(roomLimiters, k) + } + } + mu.Unlock() + } + }() + + return func(w http.ResponseWriter, r *http.Request) { + r.ParseMultipartForm(maxUploadSize) + + roomID := chi.URLParam(r, "roomID") + mu.Lock() + // no defer here becasue file upload can be slow, thus lock for too long + x, ok := roomLimiters[roomID] + if !ok { + x = roomLimiter{ + limiter: rate.NewLimiter(rate.Every(rlPeriod/time.Duration(rlCount)), rlBurst), + expire: time.Now().Add(time.Minute * 10), + } + roomLimiters[roomID] = x + } + x.expire = time.Now().Add(time.Minute * 10) + roomLimiters[roomID] = x + mu.Unlock() + if !x.limiter.Allow() { + err := errors.New(http.StatusText(http.StatusTooManyRequests)) + respondJSON(w, nil, err, http.StatusTooManyRequests) + return + } + + var ids []string + for i := 0; i < 20; i++ { + key := fmt.Sprintf("file%v", i) + file, handler, err := r.FormFile(key) + if err == http.ErrMissingFile { + break + } + if err != nil { + continue + } + defer file.Close() + b, err := ioutil.ReadAll(file) + if err != nil { + continue + } + mimeType := http.DetectContentType(b) + if mimeType == "image/gif" || mimeType == "image/jpeg" || mimeType == "image/png" { + name := handler.Filename + up, err := store.Add(name, mimeType, b) + if err != nil { + continue + } + ids = append(ids, fmt.Sprintf("%v_%v", up.ID, up.Name)) + } + } + + respondJSON(w, struct { + IDs []string `json:"ids"` + }{ids}, nil, http.StatusOK) + } +} + +// handleUploaded uploaded files display. +func handleUploaded(store *upload.Store, maxAge time.Duration) func(w http.ResponseWriter, r *http.Request) { + maxAgeHeader := fmt.Sprintf("max-age=%v", int64(maxAge/time.Second)) + return func(w http.ResponseWriter, r *http.Request) { + fileID := chi.URLParam(r, "fileID") + fileID = strings.Split(fileID, "_")[0] + up, err := store.Get(fileID) + if err != nil { + log.Println(err) + respondJSON(w, nil, err, http.StatusNotFound) + return + } + w.Header().Add("Content-Type", up.MimeType) + w.Header().Add("Content-Length", fmt.Sprint(len(up.Data))) + if maxAge > 0 { + w.Header().Add("Cache-Control", maxAgeHeader) + } + w.WriteHeader(http.StatusOK) + w.Write(up.Data) + } +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 72d35f1..af1c767 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -14,6 +14,7 @@ import ( const ( TypeTyping = "typing" TypeMessage = "message" + TypeUpload = "upload" TypePeerList = "peer.list" TypePeerInfo = "peer.info" TypePeerJoin = "peer.join" diff --git a/internal/hub/peer.go b/internal/hub/peer.go index f8ecd12..b02cf56 100644 --- a/internal/hub/peer.go +++ b/internal/hub/peer.go @@ -127,7 +127,36 @@ func (p *Peer) processMessage(b []byte) { // TODO: Respond return } - p.room.Broadcast(p.room.makeMessagePayload(msg, p), true) + p.room.Broadcast(p.room.makeMessagePayload(msg, p, m.Type), true) + + case TypeUpload: + // Check rate limits and update counters. + now := time.Now() + if p.numMessages > 0 { + if (p.numMessages%p.room.hub.cfg.RateLimitMessages+1) >= p.room.hub.cfg.RateLimitMessages && + time.Since(p.lastMessage) < p.room.hub.cfg.RateLimitInterval { + p.room.hub.Store.RemoveSession(p.ID, p.room.ID) + p.writeWSControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, TypePeerRateLimited)) + p.ws.Close() + return + } + } + p.lastMessage = now + p.numMessages++ + + msgs, ok := m.Data.([]interface{}) + if !ok { + // TODO: Respond + return + } + for _, msg := range msgs { + x, ok := msg.(string) + if !ok { + continue + } + p.room.Broadcast(p.room.makeMessagePayload(x, p, m.Type), true) + } // "Typing" status. case TypeTyping: diff --git a/internal/hub/room.go b/internal/hub/room.go index f1913ba..954628b 100644 --- a/internal/hub/room.go +++ b/internal/hub/room.go @@ -258,13 +258,13 @@ func (r *Room) makePeerUpdatePayload(p *Peer, peerUpdateType string) []byte { } // makeMessagePayload prepares a chat message. -func (r *Room) makeMessagePayload(msg string, p *Peer) []byte { +func (r *Room) makeMessagePayload(msg string, p *Peer, typ string) []byte { d := payloadMsgChat{ PeerID: p.ID, PeerHandle: p.Handle, Msg: msg, } - return r.makePayload(d, TypeMessage) + return r.makePayload(d, typ) } // makePayload prepares a message payload. diff --git a/internal/upload/store.go b/internal/upload/store.go new file mode 100644 index 0000000..a9e31d2 --- /dev/null +++ b/internal/upload/store.go @@ -0,0 +1,102 @@ +package upload + +import ( + "crypto/sha1" + "errors" + "fmt" + "sync" + "time" +) + +// Config represents the file upload options. +type Config struct { + MaxMemory string `koanf:"max-memory"` + MaxUploadSize string `koanf:"max-upload-size"` + MaxAge string `koanf:"max-age"` + RateLimitPeriod string `koanf:"rate-limit-period"` + RateLimitCount string `koanf:"rate-limit-count"` + RateLimitBurst string `koanf:"rate-limit-burst"` +} + +// Store file uploads in memory. +type Store struct { + cfg Config + maxMem int64 + mu sync.Mutex + items map[string]File + size int64 +} + +// File represents an upload. +type File struct { + CreatedAt time.Time + Data []byte + ID string + Name string + MimeType string +} + +// New returns a new file uplod store. +func New(cfg Config, maxMemory int64) *Store { + return &Store{ + cfg: cfg, + maxMem: maxMemory, + items: make(map[string]File), + } +} + +// Add a new item to the store. +func (s *Store) Add(name, mimeType string, data []byte) (File, error) { + h := sha1.New() + h.Write(data) + id := fmt.Sprintf("%x", h.Sum(nil)) + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.items[id] + if ok { + return up, nil + } + up.CreatedAt = time.Now() + up.ID = id + up.Name = name + up.MimeType = mimeType + up.Data = make([]byte, len(data), len(data)) + copy(up.Data, data) + s.items[id] = up + s.size += int64(len(data)) + for s.size > s.maxMem { + var oldest *File + for _, up := range s.items { + if oldest == nil { + oldest = &up + } else if up.CreatedAt.Before(oldest.CreatedAt) { + oldest = &up + } + } + if oldest != nil { + s.size -= int64(len(oldest.Data)) + delete(s.items, oldest.ID) + } + } + if len(s.items) < 1 { + return up, ErrFileTooLarge + } + return up, nil +} + +// Get the file with given id. +func (s *Store) Get(id string) (File, error) { + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.items[id] + if !ok { + return up, ErrFileNotFound + } + return up, nil +} + +// ErrFileNotFound indicates that the requested file was not found. +var ErrFileNotFound = errors.New("file not found") + +// ErrFileTooLarge indicates that the file was too large. +var ErrFileTooLarge = errors.New("file too large") diff --git a/main.go b/main.go index 42753b2..713f82a 100644 --- a/main.go +++ b/main.go @@ -13,17 +13,21 @@ import ( "os" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "time" + "github.com/alecthomas/units" "github.com/go-chi/chi" + "github.com/karrick/tparse/v2" "github.com/knadh/koanf" "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/providers/posflag" "github.com/knadh/niltalk/internal/hub" + "github.com/knadh/niltalk/internal/upload" "github.com/knadh/niltalk/store" "github.com/knadh/niltalk/store/fs" "github.com/knadh/niltalk/store/mem" @@ -247,6 +251,67 @@ func main() { } app.tpl = tpl + // Setup the file upload store. + var uploadCfg upload.Config + if err := ko.Unmarshal("upload", &uploadCfg); err != nil { + logger.Fatalf("error unmarshalling 'upload' config: %v", err) + } + var maxMemory int64 = 32 << 20 + if uploadCfg.MaxMemory != "" { + x, err := units.ParseStrictBytes(uploadCfg.MaxMemory) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.max-memory' config: %v", err) + } + maxMemory = x + } + + uploadStore := upload.New(uploadCfg, maxMemory) + + var maxUploadSize int64 = 32 << 20 + if uploadCfg.MaxUploadSize != "" { + x, err := units.ParseStrictBytes(uploadCfg.MaxUploadSize) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.max-upload-size' config: %v", err) + } + maxUploadSize = x + } + + var maxAge time.Duration = time.Hour * 24 * 30 * 12 + if uploadCfg.MaxAge != "" { + x, err := tparse.AbsoluteDuration(time.Now(), uploadCfg.MaxAge) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.max-age' config: %v", err) + } + maxAge = x + } + + var rlPeriod time.Duration = time.Minute + if uploadCfg.RateLimitPeriod != "" { + x, err := tparse.AbsoluteDuration(time.Now(), uploadCfg.RateLimitPeriod) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.rate-limit-period' config: %v", err) + } + rlPeriod = x + } + + rlCount := 20.0 + if uploadCfg.RateLimitCount != "" { + x, err := strconv.ParseFloat(uploadCfg.RateLimitCount, 64) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.rate-limit-count' config: %v", err) + } + rlCount = x + } + + rlBurst := 1 + if uploadCfg.RateLimitBurst != "" { + x, err := strconv.Atoi(uploadCfg.RateLimitBurst) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.rate-limit-burst' config: %v", err) + } + rlBurst = x + } + // Register HTTP routes. r := chi.NewRouter() r.Get("/", wrap(handleIndex, app, 0)) @@ -257,6 +322,9 @@ func main() { r.Delete("/api/rooms/{roomID}/login", wrap(handleLogout, app, hasAuth|hasRoom)) r.Post("/api/rooms", wrap(handleCreateRoom, app, 0)) + r.Post("/api/rooms/{roomID}/upload", handleUpload(uploadStore, maxUploadSize, rlPeriod, rlCount, rlBurst)) + r.Get("/api/rooms/{roomID}/uploaded/{fileID}", handleUploaded(uploadStore, maxAge)) + // Views. r.Get("/r/{roomID}", wrap(handleRoomPage, app, hasAuth|hasRoom)) r.Get("/static/*", func(w http.ResponseWriter, r *http.Request) { diff --git a/static/static/app.js b/static/static/app.js index 7e85e27..8ed1aa4 100644 --- a/static/static/app.js +++ b/static/static/app.js @@ -58,7 +58,10 @@ var app = new Vue({ // Chat data. self: {}, messages: [], - peers: [] + peers: [], + + // upload + isDraggingOver: false, }, created: function () { this.initClient(); @@ -383,7 +386,7 @@ var app = new Vue({ this.typingPeers.delete(data.data.peer_id); this.messages.push({ - type: Client.MsgType["message"], + type: data.type, timestamp: data.timestamp, message: data.data.message, peer: { @@ -409,6 +412,7 @@ var app = new Vue({ Client.on(Client.MsgType["peer.join"], (data) => { this.onPeerJoinLeave(data, Client.MsgType["peer.join"]); }); Client.on(Client.MsgType["peer.leave"], (data) => { this.onPeerJoinLeave(data, Client.MsgType["peer.leave"]); }); Client.on(Client.MsgType["message"], this.onMessage); + Client.on(Client.MsgType["upload"], this.onMessage); Client.on(Client.MsgType["typing"], this.onTyping); }, @@ -443,6 +447,48 @@ var app = new Vue({ this.$forceUpdate(); } }, typingDebounceInterval); + }, + + dragEnter(e) { + this.isDraggingOver=true + }, + + dragLeave(e) { + this.isDraggingOver=false + }, + + // image upload + addFile(e) { + this.isDraggingOver=false + // based on https://www.raymondcamden.com/2019/08/08/drag-and-drop-file-upload-in-vuejs + let droppedFiles = e.dataTransfer.files; + if(!droppedFiles) return; + // this tip, convert FileList to array, credit: https://www.smashingmagazine.com/2018/01/drag-drop-file-uploader-vanilla-js/ + let formData = new FormData(); + ([...droppedFiles]).forEach((f,x) => { + if (x<20) { + formData.append('file'+(x), f); + }else{ + this.notify("Too much files to upload", notifType.error); + } + }); + + fetch("/api/rooms/" + _room.id + "/upload", { + method:'POST', + body: formData + }) + .then(res => res.json()) + .then(res => { + if (res.error){ + this.notify(res.error, notifType.error); + }else{ + Client.sendMessage(Client.MsgType["upload"], res.data.ids); + } + }) + .catch(err => { + this.notify(err, notifType.error); + }); + } } }); diff --git a/static/static/client.js b/static/static/client.js index 54ea0d3..5752229 100644 --- a/static/static/client.js +++ b/static/static/client.js @@ -6,6 +6,7 @@ var Client = new function () { "room.dispose": "room.dispose", "room.full": "room.full", "message": "message", + "upload": "upload", "typing": "typing", "peer.list": "peer.list", "peer.info": "peer.info", diff --git a/static/static/style.css b/static/static/style.css index d0d8de6..08fe382 100644 --- a/static/static/style.css +++ b/static/static/style.css @@ -268,10 +268,20 @@ form .help { overflow-y: auto; padding-right: 15px; } +.chat .messages.dragover { + border-style: dashed; + border-width: 2px; + border-color: gray; +} .chat .messages .message { border-bottom: 1px solid #eee; padding: 15px 0; } +.chat .messages .message .upload { + max-width: 50%; + margin: auto; + display: block; +} .chat .messages .message:hover { background: #fafafa; } diff --git a/static/templates/room.html b/static/templates/room.html index d3a48e2..a858b62 100644 --- a/static/templates/room.html +++ b/static/templates/room.html @@ -35,7 +35,11 @@

Join room

{( sidebarOn ? "→" : "←" )} 👥{( peers.length )} -
+
  • @@ -48,6 +52,20 @@

    Join room

+
+
+ + + {( m.peer.handle )} + + {( formatDate(m.timestamp) )} +
+
+ + + +
+
{( formatDate(m.timestamp) )} — @@ -124,4 +142,4 @@

Disconnected.

--> {{ template "footer" . }} -{{end}} \ No newline at end of file +{{end}} From c09480e31a01e5878a05bf4eff4fef30c961848f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Auger?= Date: Tue, 25 Aug 2020 16:25:44 +0200 Subject: [PATCH 2/3] Add file upload support --- config.toml.sample | 9 ++++ go.mod | 3 ++ go.sum | 9 ++++ handlers.go | 108 +++++++++++++++++++++++++++++++++++++ internal/hub/hub.go | 1 + internal/hub/peer.go | 31 ++++++++++- internal/hub/room.go | 4 +- internal/upload/store.go | 102 +++++++++++++++++++++++++++++++++++ main.go | 68 +++++++++++++++++++++++ static/static/app.js | 50 ++++++++++++++++- static/static/client.js | 1 + static/static/style.css | 10 ++++ static/templates/room.html | 22 +++++++- 13 files changed, 411 insertions(+), 7 deletions(-) create mode 100644 internal/upload/store.go diff --git a/config.toml.sample b/config.toml.sample index 36000c0..f0a1897 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -60,3 +60,12 @@ prefix_session = "NIL:SESS:ROOM:%s" # FileSystem store config. # [store] # path = "db.json" + +# File upload configuration. +[upload] +max-memory="32MB" +max-upload-size="2MB" +max-age="1year" +rate-limit-count="10" +rate-limit-period="1minute" +rate-limit-burst="1" diff --git a/go.mod b/go.mod index 52a6e82..0093783 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,16 @@ module github.com/knadh/niltalk go 1.13 require ( + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/go-chi/chi v4.1.0+incompatible github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/websocket v1.4.2 + github.com/karrick/tparse/v2 v2.8.2 github.com/knadh/koanf v0.9.1 github.com/knadh/stuffbin v1.1.0 github.com/kr/pretty v0.1.0 // indirect github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 890d0a2..597a7b8 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -14,6 +16,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/karrick/tparse/v2 v2.8.2 h1:NhvrrB7nXYa0VLn0JKn9L3oG/GZN+LB/+g5QfWE30rU= +github.com/karrick/tparse/v2 v2.8.2/go.mod h1:OzmKMqNal7LYYHaO/Ie1f/wXmLWAaGKwJmxUFNQCVxg= github.com/knadh/koanf v0.9.1 h1:qfcwiF9/Z8buTJ0QXaZvOxJ6eKJmOiiWKP/PktiW5RE= github.com/knadh/koanf v0.9.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= github.com/knadh/stuffbin v1.1.0 h1:f5S5BHzZALjuJEgTIOMC9NidEnBJM7Ze6Lu1GHR/lwU= @@ -35,6 +39,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 h1:fpnn/HnJONpIu6hkXi1u/7rR0NzilgWr4T0JmWkEitk= golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -46,8 +52,11 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ix golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers.go b/handlers.go index f2f7cd4..92c74cb 100644 --- a/handlers.go +++ b/handlers.go @@ -4,13 +4,20 @@ import ( "context" "encoding/json" "errors" + "fmt" "io/ioutil" + "log" "net/http" + "strings" + "sync" + "time" "github.com/go-chi/chi" "github.com/gorilla/websocket" "github.com/knadh/niltalk/internal/hub" + "github.com/knadh/niltalk/internal/upload" "golang.org/x/crypto/bcrypt" + "golang.org/x/time/rate" ) const ( @@ -327,3 +334,104 @@ func readJSONReq(r *http.Request, o interface{}) error { } return json.Unmarshal(b, o) } + +// handleUpload handles file uploads. +func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Duration, rlCount float64, rlBurst int) func(w http.ResponseWriter, r *http.Request) { + + type roomLimiter struct { + limiter *rate.Limiter + expire time.Time + } + var mu sync.Mutex + roomLimiters := map[string]roomLimiter{} + go func() { + t := time.NewTicker(rlPeriod + (time.Minute)) + defer t.Stop() + for range t.C { + now := time.Now() + mu.Lock() + for k, r := range roomLimiters { + if r.expire.Before(now) { + delete(roomLimiters, k) + } + } + mu.Unlock() + } + }() + + return func(w http.ResponseWriter, r *http.Request) { + r.ParseMultipartForm(maxUploadSize) + + roomID := chi.URLParam(r, "roomID") + mu.Lock() + // no defer here becasue file upload can be slow, thus lock for too long + x, ok := roomLimiters[roomID] + if !ok { + x = roomLimiter{ + limiter: rate.NewLimiter(rate.Every(rlPeriod/time.Duration(rlCount)), rlBurst), + expire: time.Now().Add(time.Minute * 10), + } + roomLimiters[roomID] = x + } + x.expire = time.Now().Add(time.Minute * 10) + roomLimiters[roomID] = x + mu.Unlock() + if !x.limiter.Allow() { + err := errors.New(http.StatusText(http.StatusTooManyRequests)) + respondJSON(w, nil, err, http.StatusTooManyRequests) + return + } + + var ids []string + for i := 0; i < 20; i++ { + key := fmt.Sprintf("file%v", i) + file, handler, err := r.FormFile(key) + if err == http.ErrMissingFile { + break + } + if err != nil { + continue + } + defer file.Close() + b, err := ioutil.ReadAll(file) + if err != nil { + continue + } + mimeType := http.DetectContentType(b) + if mimeType == "image/gif" || mimeType == "image/jpeg" || mimeType == "image/png" { + name := handler.Filename + up, err := store.Add(name, mimeType, b) + if err != nil { + continue + } + ids = append(ids, fmt.Sprintf("%v_%v", up.ID, up.Name)) + } + } + + respondJSON(w, struct { + IDs []string `json:"ids"` + }{ids}, nil, http.StatusOK) + } +} + +// handleUploaded uploaded files display. +func handleUploaded(store *upload.Store, maxAge time.Duration) func(w http.ResponseWriter, r *http.Request) { + maxAgeHeader := fmt.Sprintf("max-age=%v", int64(maxAge/time.Second)) + return func(w http.ResponseWriter, r *http.Request) { + fileID := chi.URLParam(r, "fileID") + fileID = strings.Split(fileID, "_")[0] + up, err := store.Get(fileID) + if err != nil { + log.Println(err) + respondJSON(w, nil, err, http.StatusNotFound) + return + } + w.Header().Add("Content-Type", up.MimeType) + w.Header().Add("Content-Length", fmt.Sprint(len(up.Data))) + if maxAge > 0 { + w.Header().Add("Cache-Control", maxAgeHeader) + } + w.WriteHeader(http.StatusOK) + w.Write(up.Data) + } +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 72d35f1..af1c767 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -14,6 +14,7 @@ import ( const ( TypeTyping = "typing" TypeMessage = "message" + TypeUpload = "upload" TypePeerList = "peer.list" TypePeerInfo = "peer.info" TypePeerJoin = "peer.join" diff --git a/internal/hub/peer.go b/internal/hub/peer.go index f8ecd12..b02cf56 100644 --- a/internal/hub/peer.go +++ b/internal/hub/peer.go @@ -127,7 +127,36 @@ func (p *Peer) processMessage(b []byte) { // TODO: Respond return } - p.room.Broadcast(p.room.makeMessagePayload(msg, p), true) + p.room.Broadcast(p.room.makeMessagePayload(msg, p, m.Type), true) + + case TypeUpload: + // Check rate limits and update counters. + now := time.Now() + if p.numMessages > 0 { + if (p.numMessages%p.room.hub.cfg.RateLimitMessages+1) >= p.room.hub.cfg.RateLimitMessages && + time.Since(p.lastMessage) < p.room.hub.cfg.RateLimitInterval { + p.room.hub.Store.RemoveSession(p.ID, p.room.ID) + p.writeWSControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, TypePeerRateLimited)) + p.ws.Close() + return + } + } + p.lastMessage = now + p.numMessages++ + + msgs, ok := m.Data.([]interface{}) + if !ok { + // TODO: Respond + return + } + for _, msg := range msgs { + x, ok := msg.(string) + if !ok { + continue + } + p.room.Broadcast(p.room.makeMessagePayload(x, p, m.Type), true) + } // "Typing" status. case TypeTyping: diff --git a/internal/hub/room.go b/internal/hub/room.go index f1913ba..954628b 100644 --- a/internal/hub/room.go +++ b/internal/hub/room.go @@ -258,13 +258,13 @@ func (r *Room) makePeerUpdatePayload(p *Peer, peerUpdateType string) []byte { } // makeMessagePayload prepares a chat message. -func (r *Room) makeMessagePayload(msg string, p *Peer) []byte { +func (r *Room) makeMessagePayload(msg string, p *Peer, typ string) []byte { d := payloadMsgChat{ PeerID: p.ID, PeerHandle: p.Handle, Msg: msg, } - return r.makePayload(d, TypeMessage) + return r.makePayload(d, typ) } // makePayload prepares a message payload. diff --git a/internal/upload/store.go b/internal/upload/store.go new file mode 100644 index 0000000..a9e31d2 --- /dev/null +++ b/internal/upload/store.go @@ -0,0 +1,102 @@ +package upload + +import ( + "crypto/sha1" + "errors" + "fmt" + "sync" + "time" +) + +// Config represents the file upload options. +type Config struct { + MaxMemory string `koanf:"max-memory"` + MaxUploadSize string `koanf:"max-upload-size"` + MaxAge string `koanf:"max-age"` + RateLimitPeriod string `koanf:"rate-limit-period"` + RateLimitCount string `koanf:"rate-limit-count"` + RateLimitBurst string `koanf:"rate-limit-burst"` +} + +// Store file uploads in memory. +type Store struct { + cfg Config + maxMem int64 + mu sync.Mutex + items map[string]File + size int64 +} + +// File represents an upload. +type File struct { + CreatedAt time.Time + Data []byte + ID string + Name string + MimeType string +} + +// New returns a new file uplod store. +func New(cfg Config, maxMemory int64) *Store { + return &Store{ + cfg: cfg, + maxMem: maxMemory, + items: make(map[string]File), + } +} + +// Add a new item to the store. +func (s *Store) Add(name, mimeType string, data []byte) (File, error) { + h := sha1.New() + h.Write(data) + id := fmt.Sprintf("%x", h.Sum(nil)) + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.items[id] + if ok { + return up, nil + } + up.CreatedAt = time.Now() + up.ID = id + up.Name = name + up.MimeType = mimeType + up.Data = make([]byte, len(data), len(data)) + copy(up.Data, data) + s.items[id] = up + s.size += int64(len(data)) + for s.size > s.maxMem { + var oldest *File + for _, up := range s.items { + if oldest == nil { + oldest = &up + } else if up.CreatedAt.Before(oldest.CreatedAt) { + oldest = &up + } + } + if oldest != nil { + s.size -= int64(len(oldest.Data)) + delete(s.items, oldest.ID) + } + } + if len(s.items) < 1 { + return up, ErrFileTooLarge + } + return up, nil +} + +// Get the file with given id. +func (s *Store) Get(id string) (File, error) { + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.items[id] + if !ok { + return up, ErrFileNotFound + } + return up, nil +} + +// ErrFileNotFound indicates that the requested file was not found. +var ErrFileNotFound = errors.New("file not found") + +// ErrFileTooLarge indicates that the file was too large. +var ErrFileTooLarge = errors.New("file too large") diff --git a/main.go b/main.go index 42753b2..713f82a 100644 --- a/main.go +++ b/main.go @@ -13,17 +13,21 @@ import ( "os" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "time" + "github.com/alecthomas/units" "github.com/go-chi/chi" + "github.com/karrick/tparse/v2" "github.com/knadh/koanf" "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/providers/posflag" "github.com/knadh/niltalk/internal/hub" + "github.com/knadh/niltalk/internal/upload" "github.com/knadh/niltalk/store" "github.com/knadh/niltalk/store/fs" "github.com/knadh/niltalk/store/mem" @@ -247,6 +251,67 @@ func main() { } app.tpl = tpl + // Setup the file upload store. + var uploadCfg upload.Config + if err := ko.Unmarshal("upload", &uploadCfg); err != nil { + logger.Fatalf("error unmarshalling 'upload' config: %v", err) + } + var maxMemory int64 = 32 << 20 + if uploadCfg.MaxMemory != "" { + x, err := units.ParseStrictBytes(uploadCfg.MaxMemory) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.max-memory' config: %v", err) + } + maxMemory = x + } + + uploadStore := upload.New(uploadCfg, maxMemory) + + var maxUploadSize int64 = 32 << 20 + if uploadCfg.MaxUploadSize != "" { + x, err := units.ParseStrictBytes(uploadCfg.MaxUploadSize) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.max-upload-size' config: %v", err) + } + maxUploadSize = x + } + + var maxAge time.Duration = time.Hour * 24 * 30 * 12 + if uploadCfg.MaxAge != "" { + x, err := tparse.AbsoluteDuration(time.Now(), uploadCfg.MaxAge) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.max-age' config: %v", err) + } + maxAge = x + } + + var rlPeriod time.Duration = time.Minute + if uploadCfg.RateLimitPeriod != "" { + x, err := tparse.AbsoluteDuration(time.Now(), uploadCfg.RateLimitPeriod) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.rate-limit-period' config: %v", err) + } + rlPeriod = x + } + + rlCount := 20.0 + if uploadCfg.RateLimitCount != "" { + x, err := strconv.ParseFloat(uploadCfg.RateLimitCount, 64) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.rate-limit-count' config: %v", err) + } + rlCount = x + } + + rlBurst := 1 + if uploadCfg.RateLimitBurst != "" { + x, err := strconv.Atoi(uploadCfg.RateLimitBurst) + if err != nil { + logger.Fatalf("error unmarshalling 'upload.rate-limit-burst' config: %v", err) + } + rlBurst = x + } + // Register HTTP routes. r := chi.NewRouter() r.Get("/", wrap(handleIndex, app, 0)) @@ -257,6 +322,9 @@ func main() { r.Delete("/api/rooms/{roomID}/login", wrap(handleLogout, app, hasAuth|hasRoom)) r.Post("/api/rooms", wrap(handleCreateRoom, app, 0)) + r.Post("/api/rooms/{roomID}/upload", handleUpload(uploadStore, maxUploadSize, rlPeriod, rlCount, rlBurst)) + r.Get("/api/rooms/{roomID}/uploaded/{fileID}", handleUploaded(uploadStore, maxAge)) + // Views. r.Get("/r/{roomID}", wrap(handleRoomPage, app, hasAuth|hasRoom)) r.Get("/static/*", func(w http.ResponseWriter, r *http.Request) { diff --git a/static/static/app.js b/static/static/app.js index 7e85e27..8ed1aa4 100644 --- a/static/static/app.js +++ b/static/static/app.js @@ -58,7 +58,10 @@ var app = new Vue({ // Chat data. self: {}, messages: [], - peers: [] + peers: [], + + // upload + isDraggingOver: false, }, created: function () { this.initClient(); @@ -383,7 +386,7 @@ var app = new Vue({ this.typingPeers.delete(data.data.peer_id); this.messages.push({ - type: Client.MsgType["message"], + type: data.type, timestamp: data.timestamp, message: data.data.message, peer: { @@ -409,6 +412,7 @@ var app = new Vue({ Client.on(Client.MsgType["peer.join"], (data) => { this.onPeerJoinLeave(data, Client.MsgType["peer.join"]); }); Client.on(Client.MsgType["peer.leave"], (data) => { this.onPeerJoinLeave(data, Client.MsgType["peer.leave"]); }); Client.on(Client.MsgType["message"], this.onMessage); + Client.on(Client.MsgType["upload"], this.onMessage); Client.on(Client.MsgType["typing"], this.onTyping); }, @@ -443,6 +447,48 @@ var app = new Vue({ this.$forceUpdate(); } }, typingDebounceInterval); + }, + + dragEnter(e) { + this.isDraggingOver=true + }, + + dragLeave(e) { + this.isDraggingOver=false + }, + + // image upload + addFile(e) { + this.isDraggingOver=false + // based on https://www.raymondcamden.com/2019/08/08/drag-and-drop-file-upload-in-vuejs + let droppedFiles = e.dataTransfer.files; + if(!droppedFiles) return; + // this tip, convert FileList to array, credit: https://www.smashingmagazine.com/2018/01/drag-drop-file-uploader-vanilla-js/ + let formData = new FormData(); + ([...droppedFiles]).forEach((f,x) => { + if (x<20) { + formData.append('file'+(x), f); + }else{ + this.notify("Too much files to upload", notifType.error); + } + }); + + fetch("/api/rooms/" + _room.id + "/upload", { + method:'POST', + body: formData + }) + .then(res => res.json()) + .then(res => { + if (res.error){ + this.notify(res.error, notifType.error); + }else{ + Client.sendMessage(Client.MsgType["upload"], res.data.ids); + } + }) + .catch(err => { + this.notify(err, notifType.error); + }); + } } }); diff --git a/static/static/client.js b/static/static/client.js index 54ea0d3..5752229 100644 --- a/static/static/client.js +++ b/static/static/client.js @@ -6,6 +6,7 @@ var Client = new function () { "room.dispose": "room.dispose", "room.full": "room.full", "message": "message", + "upload": "upload", "typing": "typing", "peer.list": "peer.list", "peer.info": "peer.info", diff --git a/static/static/style.css b/static/static/style.css index d0d8de6..08fe382 100644 --- a/static/static/style.css +++ b/static/static/style.css @@ -268,10 +268,20 @@ form .help { overflow-y: auto; padding-right: 15px; } +.chat .messages.dragover { + border-style: dashed; + border-width: 2px; + border-color: gray; +} .chat .messages .message { border-bottom: 1px solid #eee; padding: 15px 0; } +.chat .messages .message .upload { + max-width: 50%; + margin: auto; + display: block; +} .chat .messages .message:hover { background: #fafafa; } diff --git a/static/templates/room.html b/static/templates/room.html index d3a48e2..a858b62 100644 --- a/static/templates/room.html +++ b/static/templates/room.html @@ -35,7 +35,11 @@

Join room

{( sidebarOn ? "→" : "←" )} 👥{( peers.length )} -
+
  • @@ -48,6 +52,20 @@

    Join room

+
+
+ + + {( m.peer.handle )} + + {( formatDate(m.timestamp) )} +
+
+ + + +
+
{( formatDate(m.timestamp) )} — @@ -124,4 +142,4 @@

Disconnected.

--> {{ template "footer" . }} -{{end}} \ No newline at end of file +{{end}} From bbc04e896c1ddcb7b9b562b404546bbe9b568eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Auger?= Date: Sat, 29 Aug 2020 17:47:38 +0200 Subject: [PATCH 3/3] Improve upload store initialization to clean up main --- handlers.go | 14 +++---- internal/upload/store.go | 87 +++++++++++++++++++++++++++++++++++----- main.go | 63 +++-------------------------- 3 files changed, 89 insertions(+), 75 deletions(-) diff --git a/handlers.go b/handlers.go index d10b217..b3e7c43 100644 --- a/handlers.go +++ b/handlers.go @@ -346,7 +346,7 @@ func readJSONReq(r *http.Request, o interface{}) error { } // handleUpload handles file uploads. -func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Duration, rlCount float64, rlBurst int) func(w http.ResponseWriter, r *http.Request) { +func handleUpload(store *upload.Store) func(w http.ResponseWriter, r *http.Request) { type roomLimiter struct { limiter *rate.Limiter @@ -355,7 +355,7 @@ func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Durati var mu sync.Mutex roomLimiters := map[string]roomLimiter{} go func() { - t := time.NewTicker(rlPeriod + (time.Minute)) + t := time.NewTicker(store.RlPeriod + (time.Minute)) defer t.Stop() for range t.C { now := time.Now() @@ -370,7 +370,7 @@ func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Durati }() return func(w http.ResponseWriter, r *http.Request) { - r.ParseMultipartForm(maxUploadSize) + r.ParseMultipartForm(store.MaxUploadSize) roomID := chi.URLParam(r, "roomID") mu.Lock() @@ -378,7 +378,7 @@ func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Durati x, ok := roomLimiters[roomID] if !ok { x = roomLimiter{ - limiter: rate.NewLimiter(rate.Every(rlPeriod/time.Duration(rlCount)), rlBurst), + limiter: rate.NewLimiter(rate.Every(store.RlPeriod/time.Duration(store.RlCount)), store.RlBurst), expire: time.Now().Add(time.Minute * 10), } roomLimiters[roomID] = x @@ -425,8 +425,8 @@ func handleUpload(store *upload.Store, maxUploadSize int64, rlPeriod time.Durati } // handleUploaded uploaded files display. -func handleUploaded(store *upload.Store, maxAge time.Duration) func(w http.ResponseWriter, r *http.Request) { - maxAgeHeader := fmt.Sprintf("max-age=%v", int64(maxAge/time.Second)) +func handleUploaded(store *upload.Store) func(w http.ResponseWriter, r *http.Request) { + maxAgeHeader := fmt.Sprintf("max-age=%v", int64(store.MaxAge/time.Second)) return func(w http.ResponseWriter, r *http.Request) { fileID := chi.URLParam(r, "fileID") fileID = strings.Split(fileID, "_")[0] @@ -438,7 +438,7 @@ func handleUploaded(store *upload.Store, maxAge time.Duration) func(w http.Respo } w.Header().Add("Content-Type", up.MimeType) w.Header().Add("Content-Length", fmt.Sprint(len(up.Data))) - if maxAge > 0 { + if store.MaxAge > 0 { w.Header().Add("Cache-Control", maxAgeHeader) } w.WriteHeader(http.StatusOK) diff --git a/internal/upload/store.go b/internal/upload/store.go index a9e31d2..32319f6 100644 --- a/internal/upload/store.go +++ b/internal/upload/store.go @@ -4,8 +4,12 @@ import ( "crypto/sha1" "errors" "fmt" + "strconv" "sync" "time" + + "github.com/alecthomas/units" + tparse "github.com/karrick/tparse/v2" ) // Config represents the file upload options. @@ -20,11 +24,75 @@ type Config struct { // Store file uploads in memory. type Store struct { - cfg Config - maxMem int64 - mu sync.Mutex - items map[string]File - size int64 + cfg Config + mu sync.Mutex + items map[string]File + size int64 + + MaxMemory int64 + MaxUploadSize int64 + MaxAge time.Duration + RlPeriod time.Duration + RlCount float64 + RlBurst int +} + +//Init the store, parsing configuration values. +func (s *Store) Init() error { + s.MaxMemory = 32 << 20 + if s.cfg.MaxMemory != "" { + x, err := units.ParseStrictBytes(s.cfg.MaxMemory) + if err != nil { + return fmt.Errorf("error unmarshalling 'upload.max-memory' config: %v", err) + } + s.MaxMemory = x + } + + s.MaxUploadSize = 32 << 20 + if s.cfg.MaxUploadSize != "" { + x, err := units.ParseStrictBytes(s.cfg.MaxUploadSize) + if err != nil { + return fmt.Errorf("error unmarshalling 'upload.max-upload-size' config: %v", err) + } + s.MaxUploadSize = x + } + + s.MaxAge = time.Hour * 24 * 30 * 12 + if s.cfg.MaxAge != "" { + x, err := tparse.AbsoluteDuration(time.Now(), s.cfg.MaxAge) + if err != nil { + return fmt.Errorf("error unmarshalling 'upload.max-age' config: %v", err) + } + s.MaxAge = x + } + + s.RlPeriod = time.Minute + if s.cfg.RateLimitPeriod != "" { + x, err := tparse.AbsoluteDuration(time.Now(), s.cfg.RateLimitPeriod) + if err != nil { + return fmt.Errorf("error unmarshalling 'upload.rate-limit-period' config: %v", err) + } + s.RlPeriod = x + } + + s.RlCount = 20.0 + if s.cfg.RateLimitCount != "" { + x, err := strconv.ParseFloat(s.cfg.RateLimitCount, 64) + if err != nil { + return fmt.Errorf("error unmarshalling 'upload.rate-limit-count' config: %v", err) + } + s.RlCount = x + } + + s.RlBurst = 1 + if s.cfg.RateLimitBurst != "" { + x, err := strconv.Atoi(s.cfg.RateLimitBurst) + if err != nil { + return fmt.Errorf("error unmarshalling 'upload.rate-limit-burst' config: %v", err) + } + s.RlBurst = x + } + return nil } // File represents an upload. @@ -37,11 +105,10 @@ type File struct { } // New returns a new file uplod store. -func New(cfg Config, maxMemory int64) *Store { +func New(cfg Config) *Store { return &Store{ - cfg: cfg, - maxMem: maxMemory, - items: make(map[string]File), + cfg: cfg, + items: make(map[string]File), } } @@ -64,7 +131,7 @@ func (s *Store) Add(name, mimeType string, data []byte) (File, error) { copy(up.Data, data) s.items[id] = up s.size += int64(len(data)) - for s.size > s.maxMem { + for s.size > s.MaxMemory { var oldest *File for _, up := range s.items { if oldest == nil { diff --git a/main.go b/main.go index b04485e..1c0d2ce 100644 --- a/main.go +++ b/main.go @@ -13,14 +13,11 @@ import ( "os" "os/signal" "path/filepath" - "strconv" "strings" "syscall" "time" - "github.com/alecthomas/units" "github.com/go-chi/chi" - "github.com/karrick/tparse/v2" "github.com/knadh/koanf" "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/env" @@ -267,60 +264,10 @@ func main() { if err := ko.Unmarshal("upload", &uploadCfg); err != nil { logger.Fatalf("error unmarshalling 'upload' config: %v", err) } - var maxMemory int64 = 32 << 20 - if uploadCfg.MaxMemory != "" { - x, err := units.ParseStrictBytes(uploadCfg.MaxMemory) - if err != nil { - logger.Fatalf("error unmarshalling 'upload.max-memory' config: %v", err) - } - maxMemory = x - } - - uploadStore := upload.New(uploadCfg, maxMemory) - - var maxUploadSize int64 = 32 << 20 - if uploadCfg.MaxUploadSize != "" { - x, err := units.ParseStrictBytes(uploadCfg.MaxUploadSize) - if err != nil { - logger.Fatalf("error unmarshalling 'upload.max-upload-size' config: %v", err) - } - maxUploadSize = x - } - - var maxAge time.Duration = time.Hour * 24 * 30 * 12 - if uploadCfg.MaxAge != "" { - x, err := tparse.AbsoluteDuration(time.Now(), uploadCfg.MaxAge) - if err != nil { - logger.Fatalf("error unmarshalling 'upload.max-age' config: %v", err) - } - maxAge = x - } - - var rlPeriod time.Duration = time.Minute - if uploadCfg.RateLimitPeriod != "" { - x, err := tparse.AbsoluteDuration(time.Now(), uploadCfg.RateLimitPeriod) - if err != nil { - logger.Fatalf("error unmarshalling 'upload.rate-limit-period' config: %v", err) - } - rlPeriod = x - } - rlCount := 20.0 - if uploadCfg.RateLimitCount != "" { - x, err := strconv.ParseFloat(uploadCfg.RateLimitCount, 64) - if err != nil { - logger.Fatalf("error unmarshalling 'upload.rate-limit-count' config: %v", err) - } - rlCount = x - } - - rlBurst := 1 - if uploadCfg.RateLimitBurst != "" { - x, err := strconv.Atoi(uploadCfg.RateLimitBurst) - if err != nil { - logger.Fatalf("error unmarshalling 'upload.rate-limit-burst' config: %v", err) - } - rlBurst = x + uploadStore := upload.New(uploadCfg) + if err := uploadStore.Init(); err != nil { + logger.Fatalf("error initializing upload store: %v", err) } // Register HTTP routes. @@ -333,8 +280,8 @@ func main() { r.Delete("/api/rooms/{roomID}/login", wrap(handleLogout, app, hasAuth|hasRoom)) r.Post("/api/rooms", wrap(handleCreateRoom, app, 0)) - r.Post("/api/rooms/{roomID}/upload", handleUpload(uploadStore, maxUploadSize, rlPeriod, rlCount, rlBurst)) - r.Get("/api/rooms/{roomID}/uploaded/{fileID}", handleUploaded(uploadStore, maxAge)) + r.Post("/api/rooms/{roomID}/upload", handleUpload(uploadStore)) + r.Get("/api/rooms/{roomID}/uploaded/{fileID}", handleUploaded(uploadStore)) // Views. r.Get("/r/{roomID}", wrap(handleRoomPage, app, hasAuth|hasRoom))