pixiv private isucon 2016 ��ά (3/5)
����������:
������ǧ
access.log ��⤦���ٸ�ľ���ޤ��礦��
Request by total time 74.113 0.0307140489018 GET / 70.007 0.00532696697611 GET /image/* 43.428 0.0575205298013 GET /@user 24.058 0.283035294118 GET /posts?max_created_at= 23.976 0.0522352941176 POST / 12.767 0.012987792472 GET /posts/* 6.642 0.00390476190476 POST /login ...
CPU ����ֻȤäƤ���Τϥȥåץڡ����Ǥ������쥹�ݥ󥹥������������֤Ǹ����� /image/*
���ۤȤ���¤Ӥޤ�����
�������� /image/*
���С����ץ���������Τϻߤ�ʤ��Ȥ����ޤ���
Go ��Ȥ����� nginx ��Ȥ��Τ����Τ����ˤʤ�ΤǤ�������Ū�ե������ۿ��� nginx ��Ǥ�����ۤ����ƤäȤ��ᤤ�Τǡ� nginx �򳰤��ʤ����ˤǹԤ��ޤ��礦��
nginx ������
�ͤ� ISUCON ���ʳ��� nginx ������˾ܤ����ʤ��Τǡ��Ȥꤢ��������ʴ��������ꤷ�ޤ�����
root /home/isucon/private_isu/webapp/public/; location / { try_files $uri @app; } location @app { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass http://app; }
public �ǥ��쥯�ȥ�� css �Ȥ� js �����äƤ�Τǡ�������� mkdir image
���Ƥ����� try_files
�ǥ��쥯�ƥ��֤ǥե����뤬Í��Ф�����֤����ʤ���Х�С����ץ������������ñ¤¤¤ï¿½ @app
location ��Ȥ�����ˤ��Ƥ��ޤ���
���ץ�¦�ϡ� /image/*
�˥������������ä��Ȥ��˲������֤��Ĥ��Ǥ˥ե�����˽ñ¤½Ð¤ï¿½ï¿½Î¤È¡ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½Ã¥×¥ï¿½ï¿½ï¿½ï¿½É¤ï¿½ï¿½ì¤¿ï¿½È¤ï¿½ï¿½Ë¥Õ¥ï¿½ï¿½ï¿½ï¿½ï¿½Ë½ñ¤¤¤ï¿½DB�ˤϽñ¤«¤Ê¤ï¿½ï¿½è¤¦ï¿½Ë½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½Þ¤ï¿½ï¿½ï¿½DB�˽ñ¤«¤Ê¤ï¿½ï¿½Î¤ï¿½ Disk IO ����㸺�Τ���Ǥ���
--- a/app.go +++ b/app.go @@ -78,6 +78,29 @@ func init() { store = gsm.NewMemcacheStore(memcacheClient, "isucogram_", []byte("sendagaya")) } +func writeImage(id int, mime string, data []byte) { + var ext string + switch mime { + case "image/jpeg": + ext = ".jpg" + case "image/png": + ext = ".png" + case "image/gif": + ext = ".gif" + default: + fmt.Println("Failed to write file: ", id, mime) + return + } + + fn := fmt.Sprintf("../public/image/%d%s", id, ext) + f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + panic(err) + } + f.Write(data) + f.Close() +} + func dbInitialize() { sqls := []string{ "DELETE FROM users WHERE id > 1000", @@ -652,7 +675,7 @@ func postIndex(w http.ResponseWriter, r *http.Request) { query, me.ID, mime, - filedata, + []byte(""), r.FormValue("body"), ) if eerr != nil { @@ -665,7 +688,7 @@ func postIndex(w http.ResponseWriter, r *http.Request) { fmt.Println(lerr.Error()) return } - + writeImage(int(pid), mime, filedata) http.Redirect(w, r, "/posts/"+strconv.FormatInt(pid, 10), http.StatusFound) return } @@ -695,6 +718,7 @@ func getImage(c web.C, w http.ResponseWriter, r *http.Request) { if err != nil { fmt.Println(err.Error()) } + writeImage(pid, post.Mime, post.Imgdata) return }
���꤬Â��ʤ�����
����������ˤ֤Ĥ���ޤ��������꤬Â��ʤ��ƥ٥�����������ޤ��� ��Ū�ե������ۿ��� nginx ��Ǥ���뤳�Ȥǥ�������̤����餻��ȻפäƤ����Τˡ�
¿ʬ����ǽ�����夷������˥��ꥢ�������ȤΥڡ������᤯�ʤä����꤬���߲������ΤǤ��礦���ץ��ե����顼�ǡ��٥���ޡ����� (��������̤������Ƥ������ɥ�����Â������������) �Υ�������̤�Ĵ�����Ƥߤޤ���
$ go tool pprof -inuse_space app.e27b8a http://localhost:3000/debug/pprof/heap Fetching profile from http://localhost:3000/debug/pprof/heap Saved profile in /home/isucon/pprof/pprof.app.e27b8a.localhost:3000.inuse_objects.inuse_space.002.pb.gz Entering interactive mode (type "help" for commands) (pprof) top40 -cum 319.72MB of 338.26MB total (94.52%) Dropped 338 nodes (cum <= 1.69MB) Showing top 40 nodes out of 52 (cum >= 8.34MB) flat flat% sum% cum cum% 0 0% 0% 337.76MB 99.85% runtime.goexit 0 0% 0% 336.72MB 99.54% net/http.(*conn).serve 0 0% 0% 322.22MB 95.26% github.com/zenazn/goji/web.(*mStack).newStack.func1 0 0% 0% 322.22MB 95.26% github.com/zenazn/goji/web.(*router).route 0 0% 0% 322.22MB 95.26% github.com/zenazn/goji/web/middleware.AutomaticOptions.func1 0 0% 0% 322.22MB 95.26% github.com/zenazn/goji/web/middleware.Logger.func1 0 0% 0% 322.22MB 95.26% github.com/zenazn/goji/web/middleware.Recoverer.func1 0 0% 0% 322.22MB 95.26% github.com/zenazn/goji/web/middleware.RequestID.func1 0 0% 0% 322.22MB 95.26% net/http.HandlerFunc.ServeHTTP 0 0% 0% 321.72MB 95.11% github.com/zenazn/goji/web.(*Mux).ServeHTTP 0 0% 0% 321.72MB 95.11% github.com/zenazn/goji/web.(*cStack).ServeHTTP 0 0% 0% 321.72MB 95.11% net/http.(*ServeMux).ServeHTTP 0 0% 0% 321.72MB 95.11% net/http.serverHandler.ServeHTTP 0 0% 0% 309.80MB 91.59% github.com/zenazn/goji/web.netHTTPHandlerFuncWrap.ServeHTTPC 0 0% 0% 301.29MB 89.07% net/http.(*Request).FormValue 0 0% 0% 301.29MB 89.07% net/http.(*Request).ParseMultipartForm 0 0% 0% 299.29MB 88.48% main.postIndex 0.50MB 0.15% 0.15% 298.79MB 88.33% mime/multipart.(*Reader).ReadForm 0 0% 0.15% 298.29MB 88.18% bytes.(*Buffer).ReadFrom 298.29MB 88.18% 88.33% 298.29MB 88.18% bytes.makeSlice 0 0% 88.33% 298.29MB 88.18% io.Copy 0 0% 88.33% 298.29MB 88.18% io.CopyN 0 0% 88.33% 298.29MB 88.18% io.copyBuffer 0 0% 88.33% 15MB 4.44% net/http.(*conn).readRequest 1MB 0.3% 88.63% 15MB 4.44% net/http.readRequest 0 0% 88.63% 12.42MB 3.67% github.com/zenazn/goji/web.handlerFuncWrap.ServeHTTPC 10.50MB 3.11% 91.73% 11MB 3.25% net/textproto.(*Reader).ReadMIMEHeader 0 0% 91.73% 10.50MB 3.10% github.com/bradleypeabody/gorilla-sessions-memcache.(*MemcacheStore).Get 0 0% 91.73% 10.50MB 3.10% main.getSession 0 0% 91.73% 9.42MB 2.78% database/sql.(*Rows).Next 9.42MB 2.78% 94.52% 9.42MB 2.78% github.com/go-sql-driver/mysql.(*buffer).fill 0 0% 94.52% 9.42MB 2.78% github.com/go-sql-driver/mysql.(*buffer).readNext 0 0% 94.52% 9.42MB 2.78% github.com/go-sql-driver/mysql.(*mysqlConn).readPacket 0 0% 94.52% 9.42MB 2.78% github.com/go-sql-driver/mysql.(*textRows).Next 0 0% 94.52% 9.42MB 2.78% github.com/go-sql-driver/mysql.(*textRows).readRow 0 0% 94.52% 8.34MB 2.47% github.com/jmoiron/sqlx.(*DB).Get 0 0% 94.52% 8.34MB 2.47% github.com/jmoiron/sqlx.(*Row).Scan 0 0% 94.52% 8.34MB 2.47% github.com/jmoiron/sqlx.(*Row).scanAny 0 0% 94.52% 8.34MB 2.47% github.com/jmoiron/sqlx.Get 0 0% 94.52% 8.34MB 2.47% main.getImage (pprof) list ReadForm Total: 338.26MB ROUTINE ======================== mime/multipart.(*Reader).ReadForm in /home/isucon/.local/go/src/mime/multipart/formdata.go 512.02kB 298.79MB (flat, cum) 88.33% of Total . . 19:// ReadForm parses an entire multipart message whose parts have . . 20:// a Content-Disposition of "form-data". . . 21:// It stores up to maxMemory bytes of the file parts in memory . . 22:// and the remainder on disk in temporary files. . . 23:func (r *Reader) ReadForm(maxMemory int64) (f *Form, err error) { 512.02kB 512.02kB 24: form := &Form{make(map[string][]string), make(map[string][]*FileHeader)} . . 25: defer func() { . . 26: if err != nil { . . 27: form.RemoveAll() . . 28: } . . 29: }() . . 30: . . 31: maxValueBytes := int64(10 << 20) // 10 MB is a lot of text. . . 32: for { . . 33: p, err := r.NextPart() . . 34: if err == io.EOF { . . 35: break . . 36: } . . 37: if err != nil { . . 38: return nil, err . . 39: } . . 40: . . 41: name := p.FormName() . . 42: if name == "" { . . 43: continue . . 44: } . . 45: filename := p.FileName() . . 46: . . 47: var b bytes.Buffer . . 48: . . 49: if filename == "" { . . 50: // value, store as string in memory . . 51: n, err := io.CopyN(&b, p, maxValueBytes) . . 52: if err != nil && err != io.EOF { . . 53: return nil, err . . 54: } . . 55: maxValueBytes -= n . . 56: if maxValueBytes == 0 { . . 57: return nil, errors.New("multipart: message too large") . . 58: } . . 59: form.Value[name] = append(form.Value[name], b.String()) . . 60: continue . . 61: } . . 62: . . 63: // file, store in memory or on disk . . 64: fh := &FileHeader{ . . 65: Filename: filename, . . 66: Header: p.Header, . . 67: } . 298.29MB 68: n, err := io.CopyN(&b, p, maxMemory+1) . . 69: if err != nil && err != io.EOF { . . 70: return nil, err . . 71: } . . 72: if n > maxMemory { . . 73: // too big, write to disk and flush buffer (pprof) list postIndex Total: 338.26MB ROUTINE ======================== main.postIndex in /home/isucon/private_isu/webapp/golang/app.go 0 299.29MB (flat, cum) 88.48% of Total . . 612: Me User . . 613: }{p, me}) . . 614:} . . 615: . . 616:func postIndex(w http.ResponseWriter, r *http.Request) { . 512.17kB 617: me := getSessionUser(r) . . 618: if !isLogin(me) { . . 619: http.Redirect(w, r, "/login", http.StatusFound) . . 620: return . . 621: } . . 622: . 298.79MB 623: if r.FormValue("csrf_token") != getCSRFToken(r) { . . 624: w.WriteHeader(StatusUnprocessableEntity) . . 625: return . . 626: }
���꡼��ȤäƤ�ս�Ϥ狼��ޤ��������ɤ����Ƥ�꡼���ǤϤʤ��Ǥ���
�ե����륢�åץ����ɤθ�Ψ��
�����ä��ꡢmultipart �� ReadForm ���դΥ����������ɤ��ɤ�ǡ�������Ĵ�١��к���ͤ��ޤ���
����:
postIndex
�Ǻǽ��Request.FormValue
��Ƥ���Ȥ��˥ե��������Ϥ���������¹Ԥ���Ƥ��ơ������ǥ��åץ����ɤ��줿�ե�������ɤ߹���Ǥ��롣- ���åץ����ɤ��줿�ե�����ϡ��������� (�ǥե����32MB) �ޤǤ�
bytes.Buffer
���ɤ߹��ޤ졢�����Ķ������ϥƥ�ݥ��ե�����˽ñ¤½Ð¤ï¿½ï¿½ï¿½ë¡£ bytes.Buffer
�Ͼ���������������ɤ�ɤ�ꥢ�������Ȥ��Ƴ��礷�Ƥ����Τǡ�û���֤����̤Υ��������Ȥ�ȯ�����롣���ץ����ǽ���夬��Τ�Ϣư���ƥե�����Υ��åץ����ɤ����٤����������������ȤΥڡ�����®������GC���ɤ��Ĥ��ʤ��ʤä��褦����
�к�:
- multipart ����Ϥ���Ȥ�������Ū��
ParseMultipartForm
��ƤӽФ���32MB�����ä����˥ƥ�ݥ��ե������Ȥ��褦�ˤ��롣 - ����˥ե����륢�åץ����ɤβ��Ϥ�����ʤ��褦�� Mutex ���ݸ�롣
--- a/app.go +++ b/app.go @@ -17,6 +17,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "github.com/bradfitz/gomemcache/memcache" @@ -613,6 +614,8 @@ func getPostsID(c web.C, w http.ResponseWriter, r *http.Request) { }{p, me}) } +var uploadM sync.Mutex + func postIndex(w http.ResponseWriter, r *http.Request) { me := getSessionUser(r) if !isLogin(me) { @@ -620,6 +623,9 @@ func postIndex(w http.ResponseWriter, r *http.Request) { return } + uploadM.Lock() + defer uploadM.Unlock() + r.ParseMultipartForm(1 << 10) if r.FormValue("csrf_token") != getCSRFToken(r) { w.WriteHeader(StatusUnprocessableEntity) return
����Ǥ�ޤ�Â��ޤ���Ǥ������ʤΤǡ���ö������֤��Ƥ���ե�����˽ñ¤½Ð¤ï¿½ï¿½Î¤ï¿½ß¤á¡¢ï¿½ï¿½ï¿½ï¿½Õ¥ï¿½ï¿½ï¿½ï¿½ï¿½Ë¥ï¿½ï¿½Ô¡ï¿½ï¿½ï¿½ï¿½Æ¡ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ image/
�ǥ��쥯�ȥ��۲��˥�͡��ह��褦�˲������ޤ�����
--- a/app.go +++ b/app.go @@ -80,26 +80,21 @@ func init() { } func writeImage(id int, mime string, data []byte) { - var ext string - switch mime { - case "image/jpeg": - ext = ".jpg" - case "image/png": - ext = ".png" - case "image/gif": - ext = ".gif" - default: - fmt.Println("Failed to write file: ", id, mime) - return + fn := imagePath(id, mime) + err := ioutil.WriteFile(fn, data, 0666) + if err != nil { + log.Println("failed to write file; path=%q, err=%v", fn, err) } +} - fn := fmt.Sprintf("../public/image/%d%s", id, ext) - f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - panic(err) +func copyImage(id int, src, mime string) { + dst := imagePath(id, mime) + if err := os.Chmod(src, 0666); err != nil { + log.Println("failed to chmod: path=%v, %v", src, err) + } + if err := os.Rename(src, dst); err != nil { + log.Println("failed to rename; src=%q, dst=%q; %v", src, dst, err) } - f.Write(data) - f.Close() } func dbInitialize() { @@ -260,6 +255,19 @@ func imageURL(p Post) string { return "/image/" + strconv.Itoa(p.ID) + ext } +func imagePath(id int, mime string) string { + var ext string + switch mime { + case "image/jpeg": + ext = ".jpg" + case "image/png": + ext = ".png" + case "image/gif": + ext = ".gif" + } + return fmt.Sprintf("../public/image/%d%s", id, ext) +} + func isLogin(u User) bool { return u.ID != 0 } @@ -662,16 +670,20 @@ func postIndex(w http.ResponseWriter, r *http.Request) { } } - filedata, rerr := ioutil.ReadAll(file) - if rerr != nil { - fmt.Println(rerr.Error()) + tf, err := ioutil.TempFile("../upload", "img-") + if err != nil { + log.Panicf("failed to create image: %v", err) } - - if len(filedata) > UploadLimit { + written, err := io.CopyN(tf, file, UploadLimit+1) + if err != nil && err != io.EOF { + log.Panicf("failed to write to temporary file: %v", err) + } + if written > UploadLimit { + os.Remove(tf.Name()) + tf.Close() session := getSession(r) session.Values["notice"] = "�ե����륵�������ç¤ï¿½ï¿½ï¿½ï¿½ï¿½Þ¤ï¿½" session.Save(r, w) - http.Redirect(w, r, "/", http.StatusFound) return } @@ -694,7 +706,8 @@ func postIndex(w http.ResponseWriter, r *http.Request) { fmt.Println(lerr.Error()) return } - writeImage(int(pid), mime, filedata) + tf.Close() + copyImage(int(pid), tf.Name(), mime) http.Redirect(w, r, "/posts/"+strconv.FormatInt(pid, 10), http.StatusFound) return }
����ǥ�����Â������å¤ï¿½ï¿½ï¿½È¤ê¤¢ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½Åªï¿½Õ¥ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½Æ²ï¿½ï¿½ï¿½ï¿½ï¿½ nginx �����֤�����ǥ٥�����������ޤ�����
{"pass":true,"score":32771,"success":27352,"fail":0,"messages":[]}
�����ϡ� http.Request �� Form ���Ϥˤ�äƺ�������ե����뤫�饢�ץꥱ�������ΰ���ե�����ؤΥ��ԡ���̵�̤ʤΤǡ� Request �� Form �Ϥ�API��Ȥ鷺ľ�� Body ����Ϥ���Ȥ�äȸ�Ψ���ɤ��ʤ�ޤ������񤷤�����ʹ�����뤫�⤷��ޤ��󤬡��ۤ� Request.ParseMultipartForm
�Υ��ԥڤǹԤ���Ϥ��Ǥ���
�����������λ����ǥ�����Â�ϼ��ޤä��Τǡ�����ʾ�ϥǥ������ñ¤¹ï¿½ï¿½ß¤ï¿½ï¿½Í¥Ã¥ï¿½ï¿½Ë¤Ê¤Ã¤Æ¤É¤ï¿½ï¿½ï¿½ï¿½è¤¦ï¿½ï¿½Ê¤ï¿½ï¿½Ê¤ï¿½Þ¤ï¿½ï¿½Ö¤ï¿½ï¿½Æ¤ï¿½ï¿½ï¿½ï¿½Æ¡ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ê¤¿ï¿½ï¿½ï¿½Ã¤ï¿½ï¿½ï¿½ï¿½å¡¼ï¿½Ë¥ó¥°¤Ë¤ï¿½É¤ï¿½Þ¤ï¿½ï¿½ç¤¦ï¿½ï¿½
�ޤȤ�
������: 4745 (�������) -> 30076 (����) -> 32771 (nginx�Dz����ۿ�)
�פäƤ��������ǽ���夬���ʤ��Ǥ�������ϼ���˻����ۤ��Ǥ���
����ϥե����륢�åץ����ɤˤ�������Â��Ǻ�ޤ���ޤ�������(2010ǯ����Ⱦ�ˤʤä�)1GB����Υޥ���ǡ����������̤˥��åץ����ɤ���������ۤ��Τ� ISUCON ���Ȥ��꤬���Ǥ��������ȸ�Ψ�Τ����ե����륢�åץ����ɤΤ������Ĵ�٤Ƥ����ޤ��礦��
�ޤ�����ʬ���Ȥ�ͽ��θ���ǥ�����Â��Ĵ��������ˡ��Ĵ�٤Ƥ����ޤ��礦��(�㤨�� Python �ʤ�ɸ��饤�֥��� tracemalloc ���Ȥ���Ȼפ��ޤ���)
@methane