2016ǯ06��01��

pixiv private isucon 2016 ��ά (3/5)

¤Ï¤Æ¤Ê¥Ö¥Ã¥¯¥Þ¡¼¥¯¤ËÅÐÏ¿

����������:

������ǧ

access.log ��⤦���ٸ�ľ���ޤ��礦��

Request by total time
74.113 0.0307140489018 GET /
70.007 0.00532696697611 GET /image/*
43.428 0.0575205298013 GET /@user
24.058 0.283035294118 GET /posts?max_created_at=
23.976 0.0522352941176 POST /
12.767 0.012987792472 GET /posts/*
6.642 0.00390476190476 POST /login
...

CPU ����ֻȤäƤ���Τϥȥåץڡ����Ǥ������쥹�ݥ󥹥������������֤Ǹ����� /image/* ���ۤȤ���¤Ӥޤ����� �������� /image/* ���С����ץ���������Τϻߤ�ʤ��Ȥ����ޤ���

Go ��Ȥ����� nginx ��Ȥ��Τ����Τ����ˤʤ�ΤǤ�������Ū�ե������ۿ��� nginx ��Ǥ�����ۤ����ƤäȤ��ᤤ�Τǡ� nginx �򳰤��ʤ����ˤǹԤ��ޤ��礦��

nginx ������

�ͤ� ISUCON ���ʳ��� nginx ������˾ܤ����ʤ��Τǡ��Ȥꤢ��������ʴ��������ꤷ�ޤ�����

  root /home/isucon/private_isu/webapp/public/;

  location / {
    try_files $uri @app;
  }
  location @app {
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_pass http://app;
  }

public �ǥ��쥯�ȥ�� css �Ȥ� js �����äƤ�Τǡ�������� mkdir image ���Ƥ����� try_files �ǥ��쥯�ƥ��֤ǥե����뤬ͭ��Ф�����֤����ʤ���Х�С����ץ������������񤤤� @app location ��Ȥ�����ˤ��Ƥ��ޤ���

���ץ�¦�ϡ� /image/* �˥������������ä��Ȥ��˲������֤��Ĥ��Ǥ˥ե�����˽񤭽Ф��Τȡ����������åץ����ɤ��줿�Ȥ��˥ե�����˽񤤤�DB�ˤϽ񤫤ʤ��褦�˽������ޤ���DB�˽񤫤ʤ��Τ� Disk IO ����㸺�Τ���Ǥ���

--- a/app.go
+++ b/app.go
@@ -78,6 +78,29 @@ func init() {
        store = gsm.NewMemcacheStore(memcacheClient, "isucogram_", []byte("sendagaya"))
 }

+func writeImage(id int, mime string, data []byte) {
+       var ext string
+       switch mime {
+       case "image/jpeg":
+               ext = ".jpg"
+       case "image/png":
+               ext = ".png"
+       case "image/gif":
+               ext = ".gif"
+       default:
+               fmt.Println("Failed to write file: ", id, mime)
+               return
+       }
+
+       fn := fmt.Sprintf("../public/image/%d%s", id, ext)
+       f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE, 0666)
+       if err != nil {
+               panic(err)
+       }
+       f.Write(data)
+       f.Close()
+}
+
 func dbInitialize() {
        sqls := []string{
                "DELETE FROM users WHERE id > 1000",
@@ -652,7 +675,7 @@ func postIndex(w http.ResponseWriter, r *http.Request) {
                query,
                me.ID,
                mime,
-               filedata,
+               []byte(""),
                r.FormValue("body"),
        )
        if eerr != nil {
@@ -665,7 +688,7 @@ func postIndex(w http.ResponseWriter, r *http.Request) {
                fmt.Println(lerr.Error())
                return
        }
-
+       writeImage(int(pid), mime, filedata)
        http.Redirect(w, r, "/posts/"+strconv.FormatInt(pid, 10), http.StatusFound)
        return
 }
@@ -695,6 +718,7 @@ func getImage(c web.C, w http.ResponseWriter, r *http.Request) {
                if err != nil {
                        fmt.Println(err.Error())
                }
+               writeImage(pid, post.Mime, post.Imgdata)
                return
        }

���꤬­��ʤ�����

����������ˤ֤Ĥ���ޤ��������꤬­��ʤ��ƥ٥�����������ޤ��� ��Ū�ե������ۿ��� nginx ��Ǥ���뤳�Ȥǥ�������̤����餻��ȻפäƤ����Τˡ�

¿ʬ����ǽ�����夷������˥��ꥢ�������ȤΥڡ������᤯�ʤä����꤬���߲������ΤǤ��礦���ץ��ե����顼�ǡ��٥���ޡ����� (��������̤������Ƥ������ɥ�����­������������) �Υ�������̤�Ĵ�����Ƥߤޤ���

$ go tool pprof -inuse_space app.e27b8a http://localhost:3000/debug/pprof/heap
Fetching profile from http://localhost:3000/debug/pprof/heap
Saved profile in /home/isucon/pprof/pprof.app.e27b8a.localhost:3000.inuse_objects.inuse_space.002.pb.gz
Entering interactive mode (type "help" for commands)
(pprof) top40 -cum
319.72MB of 338.26MB total (94.52%)
Dropped 338 nodes (cum <= 1.69MB)
Showing top 40 nodes out of 52 (cum >= 8.34MB)
      flat  flat%   sum%        cum   cum%
         0     0%     0%   337.76MB 99.85%  runtime.goexit
         0     0%     0%   336.72MB 99.54%  net/http.(*conn).serve
         0     0%     0%   322.22MB 95.26%  github.com/zenazn/goji/web.(*mStack).newStack.func1
         0     0%     0%   322.22MB 95.26%  github.com/zenazn/goji/web.(*router).route
         0     0%     0%   322.22MB 95.26%  github.com/zenazn/goji/web/middleware.AutomaticOptions.func1
         0     0%     0%   322.22MB 95.26%  github.com/zenazn/goji/web/middleware.Logger.func1
         0     0%     0%   322.22MB 95.26%  github.com/zenazn/goji/web/middleware.Recoverer.func1
         0     0%     0%   322.22MB 95.26%  github.com/zenazn/goji/web/middleware.RequestID.func1
         0     0%     0%   322.22MB 95.26%  net/http.HandlerFunc.ServeHTTP
         0     0%     0%   321.72MB 95.11%  github.com/zenazn/goji/web.(*Mux).ServeHTTP
         0     0%     0%   321.72MB 95.11%  github.com/zenazn/goji/web.(*cStack).ServeHTTP
         0     0%     0%   321.72MB 95.11%  net/http.(*ServeMux).ServeHTTP
         0     0%     0%   321.72MB 95.11%  net/http.serverHandler.ServeHTTP
         0     0%     0%   309.80MB 91.59%  github.com/zenazn/goji/web.netHTTPHandlerFuncWrap.ServeHTTPC
         0     0%     0%   301.29MB 89.07%  net/http.(*Request).FormValue
         0     0%     0%   301.29MB 89.07%  net/http.(*Request).ParseMultipartForm
         0     0%     0%   299.29MB 88.48%  main.postIndex
    0.50MB  0.15%  0.15%   298.79MB 88.33%  mime/multipart.(*Reader).ReadForm
         0     0%  0.15%   298.29MB 88.18%  bytes.(*Buffer).ReadFrom
  298.29MB 88.18% 88.33%   298.29MB 88.18%  bytes.makeSlice
         0     0% 88.33%   298.29MB 88.18%  io.Copy
         0     0% 88.33%   298.29MB 88.18%  io.CopyN
         0     0% 88.33%   298.29MB 88.18%  io.copyBuffer
         0     0% 88.33%       15MB  4.44%  net/http.(*conn).readRequest
       1MB   0.3% 88.63%       15MB  4.44%  net/http.readRequest
         0     0% 88.63%    12.42MB  3.67%  github.com/zenazn/goji/web.handlerFuncWrap.ServeHTTPC
   10.50MB  3.11% 91.73%       11MB  3.25%  net/textproto.(*Reader).ReadMIMEHeader
         0     0% 91.73%    10.50MB  3.10%  github.com/bradleypeabody/gorilla-sessions-memcache.(*MemcacheStore).Get
         0     0% 91.73%    10.50MB  3.10%  main.getSession
         0     0% 91.73%     9.42MB  2.78%  database/sql.(*Rows).Next
    9.42MB  2.78% 94.52%     9.42MB  2.78%  github.com/go-sql-driver/mysql.(*buffer).fill
         0     0% 94.52%     9.42MB  2.78%  github.com/go-sql-driver/mysql.(*buffer).readNext
         0     0% 94.52%     9.42MB  2.78%  github.com/go-sql-driver/mysql.(*mysqlConn).readPacket
         0     0% 94.52%     9.42MB  2.78%  github.com/go-sql-driver/mysql.(*textRows).Next
         0     0% 94.52%     9.42MB  2.78%  github.com/go-sql-driver/mysql.(*textRows).readRow
         0     0% 94.52%     8.34MB  2.47%  github.com/jmoiron/sqlx.(*DB).Get
         0     0% 94.52%     8.34MB  2.47%  github.com/jmoiron/sqlx.(*Row).Scan
         0     0% 94.52%     8.34MB  2.47%  github.com/jmoiron/sqlx.(*Row).scanAny
         0     0% 94.52%     8.34MB  2.47%  github.com/jmoiron/sqlx.Get
         0     0% 94.52%     8.34MB  2.47%  main.getImage
(pprof) list ReadForm
Total: 338.26MB
ROUTINE ======================== mime/multipart.(*Reader).ReadForm in /home/isucon/.local/go/src/mime/multipart/formdata.go
  512.02kB   298.79MB (flat, cum) 88.33% of Total
         .          .     19:// ReadForm parses an entire multipart message whose parts have
         .          .     20:// a Content-Disposition of "form-data".
         .          .     21:// It stores up to maxMemory bytes of the file parts in memory
         .          .     22:// and the remainder on disk in temporary files.
         .          .     23:func (r *Reader) ReadForm(maxMemory int64) (f *Form, err error) {
  512.02kB   512.02kB     24:   form := &Form{make(map[string][]string), make(map[string][]*FileHeader)}
         .          .     25:   defer func() {
         .          .     26:           if err != nil {
         .          .     27:                   form.RemoveAll()
         .          .     28:           }
         .          .     29:   }()
         .          .     30:
         .          .     31:   maxValueBytes := int64(10 << 20) // 10 MB is a lot of text.
         .          .     32:   for {
         .          .     33:           p, err := r.NextPart()
         .          .     34:           if err == io.EOF {
         .          .     35:                   break
         .          .     36:           }
         .          .     37:           if err != nil {
         .          .     38:                   return nil, err
         .          .     39:           }
         .          .     40:
         .          .     41:           name := p.FormName()
         .          .     42:           if name == "" {
         .          .     43:                   continue
         .          .     44:           }
         .          .     45:           filename := p.FileName()
         .          .     46:
         .          .     47:           var b bytes.Buffer
         .          .     48:
         .          .     49:           if filename == "" {
         .          .     50:                   // value, store as string in memory
         .          .     51:                   n, err := io.CopyN(&b, p, maxValueBytes)
         .          .     52:                   if err != nil && err != io.EOF {
         .          .     53:                           return nil, err
         .          .     54:                   }
         .          .     55:                   maxValueBytes -= n
         .          .     56:                   if maxValueBytes == 0 {
         .          .     57:                           return nil, errors.New("multipart: message too large")
         .          .     58:                   }
         .          .     59:                   form.Value[name] = append(form.Value[name], b.String())
         .          .     60:                   continue
         .          .     61:           }
         .          .     62:
         .          .     63:           // file, store in memory or on disk
         .          .     64:           fh := &FileHeader{
         .          .     65:                   Filename: filename,
         .          .     66:                   Header:   p.Header,
         .          .     67:           }
         .   298.29MB     68:           n, err := io.CopyN(&b, p, maxMemory+1)
         .          .     69:           if err != nil && err != io.EOF {
         .          .     70:                   return nil, err
         .          .     71:           }
         .          .     72:           if n > maxMemory {
         .          .     73:                   // too big, write to disk and flush buffer
(pprof) list postIndex
Total: 338.26MB
ROUTINE ======================== main.postIndex in /home/isucon/private_isu/webapp/golang/app.go
         0   299.29MB (flat, cum) 88.48% of Total
         .          .    612:           Me   User
         .          .    613:   }{p, me})
         .          .    614:}
         .          .    615:
         .          .    616:func postIndex(w http.ResponseWriter, r *http.Request) {
         .   512.17kB    617:   me := getSessionUser(r)
         .          .    618:   if !isLogin(me) {
         .          .    619:           http.Redirect(w, r, "/login", http.StatusFound)
         .          .    620:           return
         .          .    621:   }
         .          .    622:
         .   298.79MB    623:   if r.FormValue("csrf_token") != getCSRFToken(r) {
         .          .    624:           w.WriteHeader(StatusUnprocessableEntity)
         .          .    625:           return
         .          .    626:   }

���꡼��ȤäƤ�ս�Ϥ狼��ޤ��������ɤ����Ƥ�꡼���ǤϤʤ��Ǥ���

�ե����륢�åץ����ɤθ�Ψ��

�����ä��ꡢmultipart �� ReadForm ���դΥ����������ɤ��ɤ�ǡ�������Ĵ�١��к���ͤ��ޤ���

����:

  • postIndex �Ǻǽ�� Request.FormValue ��Ƥ���Ȥ��˥ե��������Ϥ���������¹Ԥ���Ƥ��ơ������ǥ��åץ����ɤ��줿�ե�������ɤ߹���Ǥ��롣
  • ���åץ����ɤ��줿�ե�����ϡ��������� (�ǥե����32MB) �ޤǤ� bytes.Buffer ���ɤ߹��ޤ졢�����Ķ������ϥƥ�ݥ��ե�����˽ñ¤­½Ð¤ï¿½ï¿½ï¿½ë¡£
  • bytes.Buffer �Ͼ���������������ɤ�ɤ�ꥢ�������Ȥ��Ƴ��礷�Ƥ����Τǡ�û���֤����̤Υ��������Ȥ�ȯ�����롣���ץ����ǽ���夬��Τ�Ϣư���ƥե�����Υ��åץ����ɤ����٤����������������ȤΥڡ�����®������GC���ɤ��Ĥ��ʤ��ʤä��褦����

�к�:

  • multipart ����Ϥ���Ȥ�������Ū�� ParseMultipartForm ��ƤӽФ���32MB�����ä����˥ƥ�ݥ��ե������Ȥ��褦�ˤ��롣
  • ����˥ե����륢�åץ����ɤβ��Ϥ�����ʤ��褦�� Mutex ���ݸ�롣
--- a/app.go
+++ b/app.go
@@ -17,6 +17,7 @@ import (
        "regexp"
        "strconv"
        "strings"
+       "sync"
        "time"

        "github.com/bradfitz/gomemcache/memcache"
@@ -613,6 +614,8 @@ func getPostsID(c web.C, w http.ResponseWriter, r *http.Request) {
        }{p, me})
 }

+var uploadM sync.Mutex
+
 func postIndex(w http.ResponseWriter, r *http.Request) {
        me := getSessionUser(r)
        if !isLogin(me) {
@@ -620,6 +623,9 @@ func postIndex(w http.ResponseWriter, r *http.Request) {
                return
        }

+       uploadM.Lock()
+       defer uploadM.Unlock()
+       r.ParseMultipartForm(1 << 10)
        if r.FormValue("csrf_token") != getCSRFToken(r) {
                w.WriteHeader(StatusUnprocessableEntity)
                return

����Ǥ�ޤ�­��ޤ���Ǥ������ʤΤǡ���ö������֤��Ƥ���ե�����˽񤭽Ф��Τ�ߤᡢ����ե�����˥��ԡ����ơ������ image/ �ǥ��쥯�ȥ��۲��˥�͡��ह��褦�˲������ޤ�����

--- a/app.go
+++ b/app.go
@@ -80,26 +80,21 @@ func init() {
 }

 func writeImage(id int, mime string, data []byte) {
-       var ext string
-       switch mime {
-       case "image/jpeg":
-               ext = ".jpg"
-       case "image/png":
-               ext = ".png"
-       case "image/gif":
-               ext = ".gif"
-       default:
-               fmt.Println("Failed to write file: ", id, mime)
-               return
+       fn := imagePath(id, mime)
+       err := ioutil.WriteFile(fn, data, 0666)
+       if err != nil {
+               log.Println("failed to write file; path=%q, err=%v", fn, err)
        }
+}

-       fn := fmt.Sprintf("../public/image/%d%s", id, ext)
-       f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE, 0666)
-       if err != nil {
-               panic(err)
+func copyImage(id int, src, mime string) {
+       dst := imagePath(id, mime)
+       if err := os.Chmod(src, 0666); err != nil {
+               log.Println("failed to chmod: path=%v, %v", src, err)
+       }
+       if err := os.Rename(src, dst); err != nil {
+               log.Println("failed to rename; src=%q, dst=%q; %v", src, dst, err)
        }
-       f.Write(data)
-       f.Close()
 }

 func dbInitialize() {
@@ -260,6 +255,19 @@ func imageURL(p Post) string {
        return "/image/" + strconv.Itoa(p.ID) + ext
 }

+func imagePath(id int, mime string) string {
+       var ext string
+       switch mime {
+       case "image/jpeg":
+               ext = ".jpg"
+       case "image/png":
+               ext = ".png"
+       case "image/gif":
+               ext = ".gif"
+       }
+       return fmt.Sprintf("../public/image/%d%s", id, ext)
+}
+
 func isLogin(u User) bool {
        return u.ID != 0
 }
@@ -662,16 +670,20 @@ func postIndex(w http.ResponseWriter, r *http.Request) {
                }
        }

-       filedata, rerr := ioutil.ReadAll(file)
-       if rerr != nil {
-               fmt.Println(rerr.Error())
+       tf, err := ioutil.TempFile("../upload", "img-")
+       if err != nil {
+               log.Panicf("failed to create image: %v", err)
        }
-
-       if len(filedata) > UploadLimit {
+       written, err := io.CopyN(tf, file, UploadLimit+1)
+       if err != nil && err != io.EOF {
+               log.Panicf("failed to write to temporary file: %v", err)
+       }
+       if written > UploadLimit {
+               os.Remove(tf.Name())
+               tf.Close()
                session := getSession(r)
                session.Values["notice"] = "�ե����륵�������礭�����ޤ�"
                session.Save(r, w)
-
                http.Redirect(w, r, "/", http.StatusFound)
                return
        }
@@ -694,7 +706,8 @@ func postIndex(w http.ResponseWriter, r *http.Request) {
                fmt.Println(lerr.Error())
                return
        }
-       writeImage(int(pid), mime, filedata)
+       tf.Close()
+       copyImage(int(pid), tf.Name(), mime)
        http.Redirect(w, r, "/posts/"+strconv.FormatInt(pid, 10), http.StatusFound)
        return
 }

����ǥ�����­������夭���Ȥꤢ������Ū�ե��������Ʋ����� nginx �����֤�����ǥ٥�����������ޤ�����

{"pass":true,"score":32771,"success":27352,"fail":0,"messages":[]}

�����ϡ� http.Request �� Form ���Ϥˤ�äƺ�������ե����뤫�饢�ץꥱ�������ΰ���ե�����ؤΥ��ԡ���̵�̤ʤΤǡ� Request �� Form �Ϥ�API��Ȥ鷺ľ�� Body ����Ϥ���Ȥ�äȸ�Ψ���ɤ��ʤ�ޤ������񤷤�����ʹ�����뤫�⤷��ޤ��󤬡��ۤ� Request.ParseMultipartForm�Υ��ԥڤǹԤ���Ϥ��Ǥ���

�����������λ����ǥ�����­�ϼ��ޤä��Τǡ�����ʾ�ϥǥ������񤭹��ߤ��ͥå��ˤʤäƤɤ����褦��ʤ��ʤ�ޤ��֤��Ƥ����ơ������ꤿ���ä����塼�˥󥰤ˤ�ɤ�ޤ��礦��

�ޤȤ�

������: 4745 (�������) -> 30076 (����) -> 32771 (nginx�Dz����ۿ�)

�פäƤ��������ǽ���夬���ʤ��Ǥ�������ϼ���˻����ۤ��Ǥ���

����ϥե����륢�åץ����ɤˤ�������­��Ǻ�ޤ���ޤ�������(2010ǯ����Ⱦ�ˤʤä�)1GB����Υޥ���ǡ����������̤˥��åץ����ɤ���������ۤ��Τ� ISUCON ���Ȥ��꤬���Ǥ��������ȸ�Ψ�Τ����ե����륢�åץ����ɤΤ������Ĵ�٤Ƥ����ޤ��礦��

�ޤ�����ʬ���Ȥ�ͽ��θ���ǥ�����­��Ĵ��������ˡ��Ĵ�٤Ƥ����ޤ��礦��(�㤨�� Python �ʤ�ɸ��饤�֥��� tracemalloc ���Ȥ���Ȼפ��ޤ���)


@methane

songofacandy at 15:52��Comments(0)��TrackBack(0)��ISUCON | golang

�ȥ�å��Хå�URL

���ε����˥����Ȥ���

̾��:
URL:
  ����òµ­²ï¿½: ɾ��: ��    ��
 
 
 
Blog�⸡��
�ǿ�����
Archives
���Υ֥����ˤĤ���
DSAS�Ȥϡ�KLab �����ۤ����Ѥ��Ƥ��륳��ƥ�ĥ����ӥ��Ѥ�Linux�١����Υ���ե�Ǥ�������5����Υǡ������󥿤ˤƹ��ۤ������Ѥ��Ƥ��ޤ������桹��DSAS����Ȥ��䤹�����������ˡ������Ƥ����ϤDZ��ѤǤ��뤳�Ȥ��ܻؤ��ơ��������ɤ˶Ф���Ǥ��ޤ���
���Υ֥����Ǥϡ������ DSAS �ǻȤäƤ��뵻�ѤξҲ�䡢�¸����Ƥߤ���̤���𡢥ȥ�֥�˴������ޤ줿���ηи��̤ʤɡ���������������������򿥤�ޤ��ƾҲ𤷤Ƥ��������Ȼפ��ޤ���
�ǿ�������
<%==comments[n].author%>
<% } %>