diff --git a/db/init.sh b/db/init.sh index 1e142fc..cfd7be5 100644 --- a/db/init.sh +++ b/db/init.sh @@ -16,5 +16,7 @@ if [ ! -f "$DB_DIR/isucon8q-initial-dataset.sql.gz" ]; then fi mysql -uisucon torb -e 'ALTER TABLE reservations DROP KEY event_id_and_sheet_id_idx' +mysql -uisucon torb -e 'ALTER TABLE reservations DROP KEY user_id_idx' gzip -dc "$DB_DIR/isucon8q-initial-dataset.sql.gz" | mysql -uisucon torb mysql -uisucon torb -e 'ALTER TABLE reservations ADD KEY event_id_and_sheet_id_idx (event_id, sheet_id)' +mysql -uisucon torb -e 'ALTER TABLE reservations ADD KEY user_id_idx (user_id)' diff --git a/db/schema.sql b/db/schema.sql index 30c0334..ffafdc0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -29,7 +29,8 @@ CREATE TABLE IF NOT EXISTS reservations ( user_id INTEGER UNSIGNED NOT NULL, reserved_at DATETIME(6) NOT NULL, canceled_at DATETIME(6) DEFAULT NULL, - KEY event_id_and_sheet_id_idx (event_id, sheet_id) + KEY event_id_and_sheet_id_idx (event_id, sheet_id), + KEY user_id_idx (user_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE IF NOT EXISTS administrators ( diff --git a/my.cnf b/my.cnf new file mode 100644 index 0000000..33c1127 --- /dev/null +++ b/my.cnf @@ -0,0 +1,35 @@ +# For advice on how to change settings please see +# http://dev.mysql.com/doc/refman/5.6/en/server-configuration-defaults.html + +[mysqld] +# +# Remove leading # and set to the amount of RAM for the most important data +# cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%. +# innodb_buffer_pool_size = 128M +# +# Remove leading # to turn on a very important data integrity option: logging +# changes to the binary log between backups. +# log_bin +# +# Remove leading # to set options mainly useful for reporting servers. +# The server defaults are faster for transactions and fast SELECTs. +# Adjust sizes as needed, experiment to find the optimal values. +# join_buffer_size = 128M +# sort_buffer_size = 2M +# read_rnd_buffer_size = 2M + +innodb_flush_log_at_trx_commit = 0 +innodb_flush_method=O_DIRECT + +datadir=/var/lib/mysql +socket=/var/lib/mysql/mysql.sock + +# Disabling symbolic-links is recommended to prevent assorted security risks +symbolic-links=0 + +# Recommended in standard MySQL setup +sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES + +[mysqld_safe] +log-error=/var/log/mysqld.log +pid-file=/var/run/mysqld/mysqld.pid diff --git a/mysqld.service b/mysqld.service new file mode 100644 index 0000000..871f191 --- /dev/null +++ b/mysqld.service @@ -0,0 +1,48 @@ +# +# Simple MySQL systemd service file +# +# systemd supports lots of fancy features, look here (and linked docs) for a full list: +# http://www.freedesktop.org/software/systemd/man/systemd.exec.html +# +# Note: this file ( /usr/lib/systemd/system/mysql.service ) +# will be overwritten on package upgrade, please copy the file to +# +# /etc/systemd/system/mysql.service +# +# to make needed changes. +# +# systemd-delta can be used to check differences between the two mysql.service files. +# + +[Unit] +Description=MySQL Community Server +After=network.target +After=syslog.target + +[Install] +WantedBy=multi-user.target +Alias=mysql.service + +[Service] +User=mysql +Group=mysql + +# Execute pre and post scripts as root +PermissionsStartOnly=true + +# Needed to create system tables etc. +ExecStartPre=/usr/bin/mysql-systemd-start pre + +# Start main service +ExecStart=/usr/bin/mysqld_safe --basedir=/usr + +# Don't signal startup success before a ping works +ExecStartPost=/usr/bin/mysql-systemd-start post + +# Give up if ping don't get an answer +TimeoutSec=600 + +Restart=always +PrivateTmp=false + + diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..df85dd8 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,115 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user isucon; +worker_processes 1; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 2048; + multi_accept on; + use epoll; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + log_format isucon '$time_local $msec\t$status\treqtime:$request_time\t' + 'in:$request_length\tout:$bytes_sent\trequest:$request\t' + 'acceptencoding:$http_accept_encoding\treferer:$http_referer\t' + 'ua:$http_user_agent'; + + #access_log /var/log/nginx/access.log isucon; + access_log off; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 600; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + gzip_static on; + gzip_proxied off; + + upstream app { + server 127.0.0.1:8080; + keepalive 4; + } + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + root /usr/share/nginx/html; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location ^~ /favicon.ico { + root /home/isucon/torb/webapp/static/; + expires 1y; + add_header Cache-Control public; + add_header ETag ""; + } + location ^~ /css/ { + root /home/isucon/torb/webapp/static/; + expires 1y; + add_header Cache-Control public; + add_header ETag ""; + } + location ^~ /js/ { + root /home/isucon/torb/webapp/static/; + expires 1y; + add_header Cache-Control public; + add_header ETag ""; + } + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://app; + } + } + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} + diff --git a/nginx.service b/nginx.service new file mode 100644 index 0000000..5335879 --- /dev/null +++ b/nginx.service @@ -0,0 +1,22 @@ +[Unit] +Description=The nginx HTTP and reverse proxy server +After=network.target remote-fs.target nss-lookup.target + +[Service] +Type=forking +PIDFile=/run/nginx.pid +# Nginx will fail to start if /run/nginx.pid already exists but has the wrong +# SELinux context. This might happen when running `nginx -t` from the cmdline. +# https://bugzilla.redhat.com/show_bug.cgi?id=1268621 +ExecStartPre=/usr/bin/rm -f /run/nginx.pid +ExecStartPre=/usr/sbin/nginx -t +ExecStart=/usr/sbin/nginx +ExecReload=/bin/kill -s HUP $MAINPID +KillSignal=SIGQUIT +TimeoutStopSec=5 +KillMode=process +PrivateTmp=true +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/on-start-profile b/on-start-profile new file mode 100644 index 0000000..8a152f0 --- /dev/null +++ b/on-start-profile @@ -0,0 +1,15 @@ +#!/bin/bash +set -x +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:$PATH + +rm -rf /tmp/bench +mkdir -p /tmp/bench +cd /tmp/bench +date "+%Y%m%d_%H%M%S" > start_date +sudo mv /var/log/nginx/access.log /var/log/nginx/access.log.bak +sudo kill -USR1 $(cat /var/run/nginx.pid) +(timeout 60 dstat -tam --output=dstat.csv 1>/dev/null 2>/dev/null &) +(timeout 60 pidstat -C 'nginx|mysql|torb' -hur -p ALL 1 > pidstat.log &) +(timeout 60 netstat -a -p --tcp -c > netstat.log &) +(timeout 60 myprofiler -user isucon -password isucon > myprofiler.log &) + diff --git a/sysctl.conf b/sysctl.conf new file mode 100644 index 0000000..5230ce3 --- /dev/null +++ b/sysctl.conf @@ -0,0 +1,22 @@ +# sysctl settings are defined through files in +# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/. +# +# Vendors settings live in /usr/lib/sysctl.d/. +# To override a whole file, create a new file with the same in +# /etc/sysctl.d/ and put new settings there. To override +# only specific settings, add a file with a lexically later +# name in /etc/sysctl.d/ and put new settings there. +# +# For more information, see sysctl.conf(5) and sysctl.d(5). +# +net.ipv4.tcp_max_tw_buckets = 2000000 +net.ipv4.ip_local_port_range = 10000 65535 +net.ipv4.tcp_tw_recycle = 1 +net.ipv4.tcp_tw_reuse = 1 +net.ipv4.tcp_fin_timeout = 3 +net.ipv4.tcp_rmem = 16384 131072 262144 +net.ipv4.tcp_wmem = 16384 131072 262144 +net.ipv4.tcp_mem = 2048000 4096000 4096000 + +net.core.somaxconn = 32768 +net.core.netdev_max_backlog = 8192 diff --git a/torb.go.service b/torb.go.service new file mode 100644 index 0000000..87bb5e7 --- /dev/null +++ b/torb.go.service @@ -0,0 +1,20 @@ +[Unit] +Description = isucon8 qualifier webapp in Group +After=mysqld.service + +[Service] +WorkingDirectory=/home/isucon/torb/webapp/go +EnvironmentFile=/home/isucon/torb/webapp/env.sh + +ExecStart = /home/isucon/torb/webapp/go/torb + +Restart = always +Type = simple +User = isucon +Group = isucon + +StandardOutput=null +StandardError=null + +[Install] +WantedBy = multi-user.target diff --git a/webapp/go/Makefile b/webapp/go/Makefile index 2176694..d82f337 100644 --- a/webapp/go/Makefile +++ b/webapp/go/Makefile @@ -1,12 +1,10 @@ all: build +GO := GOPATH=`pwd` go -.PHONY: clean -clean: - rm -rf torb - -deps: - gb vendor restore +.PHONY: init +init: + $(GO) get -d ./... .PHONY: build -build: - GOPATH=`pwd`:`pwd`/vendor go build -v torb +build: + $(GO) install torb diff --git a/webapp/go/run.sh b/webapp/go/run.sh new file mode 100644 index 0000000..4bd86b8 --- /dev/null +++ b/webapp/go/run.sh @@ -0,0 +1,8 @@ +export APP_PORT=5000 +export DB_DATABASE=torb +export DB_HOST=localhost +export DB_PORT=3306 +export DB_USER=isucon +export DB_PASS=isucon +./bin/torb + diff --git a/webapp/go/src/.gitignore b/webapp/go/src/.gitignore new file mode 100644 index 0000000..1fb92d9 --- /dev/null +++ b/webapp/go/src/.gitignore @@ -0,0 +1,2 @@ +github.com +golang.org diff --git a/webapp/go/src/torb/prof.go b/webapp/go/src/torb/prof.go new file mode 100644 index 0000000..686ad84 --- /dev/null +++ b/webapp/go/src/torb/prof.go @@ -0,0 +1,138 @@ +package main + +import ( + "bytes" + "log" + "net/http" + "os" + "os/exec" + "runtime" + "runtime/pprof" + "time" +) + +/* +func getInitialize(w http.ResponseWriter, r *http.Request) { + noprofile := r.URL.Query().Get("noprofile") + if noprofile == "" { + StartProfile(time.Minute) + } + ... +} +*/ + +var ( + enableProfile = false + isProfiling = false + cpuProfileFile = "/tmp/cpu.pprof" + memProfileFile = "/tmp/mem.pprof" + blockProfileFile = "/tmp/block.pprof" + onStartProfileCmd = "/home/isucon/on-start-profile" + onEndProfileCmd = "/home/isucon/on-end-profile" +) + +func callOnStartProfile() { + if _, err := os.Stat(onStartProfileCmd); os.IsNotExist(err) { + log.Println("OnStartProfile command not found:", err) + return + } + cmd := exec.Command(onStartProfileCmd) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + log.Println("OnStartProfile command error:", err) + } + log.Printf("OnStartProfile Output: %s\n", out.String()) +} + +func callOnEndProfile() { + if _, err := os.Stat(onEndProfileCmd); os.IsNotExist(err) { + log.Println("OnEndProfile command not found:", err) + return + } + cmd := exec.Command(onEndProfileCmd) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + log.Println("OnEndProfile command error:", err) + } + log.Printf("OnEndProfile Output: %s\n", out.String()) +} + +func StartProfile(duration time.Duration) error { + f, err := os.Create(cpuProfileFile) + if err != nil { + return err + } + if err := pprof.StartCPUProfile(f); err != nil { + return err + } + runtime.SetBlockProfileRate(1) + isProfiling = true + if 0 < duration.Seconds() { + go func() { + time.Sleep(duration) + err := EndProfile() + if err != nil { + log.Println(err) + } + }() + } + log.Println("Profile start") + go callOnStartProfile() + return nil +} + +func EndProfile() error { + if !isProfiling { + return nil + } + isProfiling = false + pprof.StopCPUProfile() + runtime.SetBlockProfileRate(0) + log.Println("Profile end") + defer func() { + go callOnEndProfile() + }() + + mf, err := os.Create(memProfileFile) + if err != nil { + return err + } + pprof.WriteHeapProfile(mf) + + bf, err := os.Create(blockProfileFile) + if err != nil { + return err + } + pprof.Lookup("block").WriteTo(bf, 0) + return nil +} + +func init() { + log.Println("add handler /startprof /endprof") + http.HandleFunc("/startprof", func(w http.ResponseWriter, r *http.Request) { + err := StartProfile(time.Minute) + if err != nil { + w.Write([]byte(err.Error())) + } else { + w.Write([]byte("profile started\n")) + } + }) + + http.HandleFunc("/endprof", func(w http.ResponseWriter, r *http.Request) { + err := EndProfile() + w.Write([]byte("profile ended\n")) + if err != nil { + w.Write([]byte(err.Error() + "\n")) + } + }) + + go func() { + if err := http.ListenAndServe(":8081", nil); err != nil { + log.Println("ListenAndServe: ", err) + } + }() +}