From 9b809fb2d5337896c5cf9d7e757a54741d045ebd Mon Sep 17 00:00:00 2001 From: Krzysztof Kowalczyk Date: Fri, 8 Oct 2021 22:46:41 +0000 Subject: [PATCH] start on loghttp --- go.mod | 1 + go.sum | 56 +++++++ loghttp/loghttp.go | 387 +++++++++++++++++++++++++++++++++++++++++++++ loghttp/readme.md | 5 + 4 files changed, 449 insertions(+) create mode 100644 loghttp/loghttp.go create mode 100644 loghttp/readme.md diff --git a/go.mod b/go.mod index 29fbd67..34afd5c 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/BurntSushi/toml v0.4.1 // indirect github.com/andybalholm/brotli v1.0.3 + github.com/minio/minio-go/v7 v7.0.14 // indirect github.com/stretchr/testify v1.7.0 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect diff --git a/go.sum b/go.sum index de70d34..033d681 100644 --- a/go.sum +++ b/go.sum @@ -5,9 +5,49 @@ github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0 github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.14 h1:T7cw8P586gVwEEd0y21kTYtloD576XZgP62N8pE130s= +github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -15,19 +55,27 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -35,11 +83,14 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= @@ -49,6 +100,11 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.2.1 h1:/EPr//+UMMXwMTkXvCCoaJDq8cpjMO80Ou+L4PDo2mY= diff --git a/loghttp/loghttp.go b/loghttp/loghttp.go new file mode 100644 index 0000000..f57e719 --- /dev/null +++ b/loghttp/loghttp.go @@ -0,0 +1,387 @@ +package loghttp + +import ( + "context" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/andybalholm/brotli" + "github.com/kjk/common/filerotate" + "github.com/kjk/common/siser" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + logsDirCached = "" + httpLogSiser *siser.Writer + httpLogRec siser.Record + httpLogMu sync.Mutex + httpLogApp = "" +) + +type Config struct { + Dir string + AppName string + + // defines s3-copmatible storage + // if not provided, will not upload + Secret string + Access string + Bucket string + Endpoint string +} + +func getLogsDir() string { + if logsDirCached != "" { + return logsDirCached + } + logsDirCached = "logs" + must(os.MkdirAll(logsDirCached, 0755)) + return logsDirCached +} + +// /httplog-2021-10-06_01.txt.br +// => +//apps/cheatsheet/httplog/2021/10-06/2021-10-06_01.txt.br +// return "" if is in unexpected format +func remotePathFromFilePath(path string) string { + name := filepath.Base(path) + parts := strings.Split(name, "_") + if len(parts) != 2 { + return "" + } + // parts[1]: 01.txt.br + hr := strings.Split(parts[1], ".")[0] + if len(hr) != 2 { + return "" + } + // parts[0]: httplog-2021-10-06 + parts = strings.Split(parts[0], "-") + if len(parts) != 4 { + return "" + } + year := parts[1] + month := parts[2] + day := parts[3] + name = fmt.Sprintf("%s/%s-%s/%s-%s-%s_%s.txt.br", year, month, day, year, month, day, hr) + return fmt.Sprintf("apps/%s/httplog/%s", httpLogApp, name) +} + +// upload httplog-2021-10-06_01.txt as +// apps/cheatsheet/httplog/2021/10-06/2021-10-06_01.txt.br +func uploadCompressedHTTPLog(path string) error { + pathBr := path + ".br" + createCompressed := func() error { + r, err := os.Open(path) + if err != nil { + return err + } + defer r.Close() + os.Remove(pathBr) + f, err := os.Create(pathBr) + if err != nil { + return err + } + w := brotli.NewWriterLevel(f, brotli.BestCompression) + _, err = io.Copy(w, r) + err2 := w.Close() + err3 := f.Close() + if err != nil { + return err + } + if err2 != nil { + return err2 + } + return err3 + } + defer os.Remove(pathBr) + + // timeStart := time.Now() + err := createCompressed() + if err != nil { + return err + } + /* + dur := time.Since(timeStart) + origSize := getFileSize(path) + comprSize := getFileSize(pathBr) + p := perc(origSize, comprSize) + logf(ctx(), "uploadCompressedHTTPLog: compressed '%s' as '%s', %s => %s (%.2f%%) in %s\n", path, pathBr, formatSize(origSize), formatSize(comprSize), p, dur) + */ + // timeStart = time.Now() + mc := newMinioSpacesClient() + remotePath := remotePathFromFilePath(pathBr) + if remotePath == "" { + // logf(ctx(), "uploadCompressedHTTPLog: remotePathFromFilePath() failed for '%s'\n", pathBr) + return nil + } + err = minioUploadFilePublic(mc, remotePath, pathBr) + if err != nil { + // logerrf(ctx(), "uploadCompressedHTTPLog: minioUploadFilePublic() failed with '%s'\n", err) + return nil + } + // logf(ctx(), "uploadCompressedHTTPLog: uploaded '%s' as '%s' in %s\n", pathBr, remotePath, time.Since(timeStart)) + return nil +} + +func didRotateHTTPLog(path string, didRotate bool) { + canUpload := hasSpacesCreds() + // logf(ctx(), "didRotateHTTPLog: '%s', didRotate: %v, hasSpacesCreds: %v\n", path, didRotate, canUpload) + if !canUpload || !didRotate { + return + } + go uploadCompressedHTTPLog(path) +} + +func NewLogHourly(dir string, didClose func(path string, didRotate bool)) (*filerotate.File, error) { + hourly := func(creationTime time.Time, now time.Time) string { + if filerotate.IsSameHour(creationTime, now) { + return "" + } + name := "httplog-" + now.Format("2006-01-02_15") + ".txt" + path := filepath.Join(dir, name) + // logf(ctx(), "NewLogHourly: '%s'\n", path) + return path + } + config := filerotate.Config{ + DidClose: didClose, + PathIfShouldRotate: hourly, + } + return filerotate.New(&config) +} + +func OpenHTTPLog(app string) func() { + panicIf(app == "") + dir := getLogsDir() + + logFile, err := NewLogHourly(dir, didRotateHTTPLog) + must(err) + httpLogSiser = siser.NewWriter(logFile) + // TODO: should I change filerotate so that it opens the file immedaitely? + return func() { + _ = logFile.Close() + httpLogSiser = nil + } +} + +var ( + hdrsToNotLog = []string{ + "Connection", + "Sec-Ch-Ua-Mobile", + "Sec-Fetch-Dest", + "Sec-Ch-Ua-Platform", + "Dnt", + "Upgrade-Insecure-Requests", + "Sec-Fetch-Site", + "Sec-Fetch-Mode", + "Sec-Fetch-User", + "If-Modified-Since", + "Accept-Language", + "Cf-Ray", + "CF-Visitor", + "X-Request-Start", + "Cdn-Loop", + "X-Forwarded-Proto", + } + hdrsToNotLogMap map[string]bool +) + +func shouldLogHeader(s string) bool { + if hdrsToNotLogMap == nil { + hdrsToNotLogMap = map[string]bool{} + for _, h := range hdrsToNotLog { + h = strings.ToLower(h) + hdrsToNotLogMap[h] = true + } + } + s = strings.ToLower(s) + return !hdrsToNotLogMap[s] +} + +func recWriteNonEmpty(rec *siser.Record, k, v string) { + if v != "" { + rec.Write(k, v) + } +} + +func LogHTTPReq(r *http.Request, code int, size int64, dur time.Duration) error { + uri := r.URL.Path + if strings.HasPrefix(uri, "/ping") { + // our internal health monitoring endpoint is called frequently, don't log + return nil + } + + httpLogMu.Lock() + defer httpLogMu.Unlock() + + if httpLogSiser == nil { + return nil + } + + rec := &httpLogRec + rec.Reset() + rec.Write("req", fmt.Sprintf("%s %s %d", r.Method, r.RequestURI, code)) + recWriteNonEmpty(rec, "host", r.Host) + rec.Write("ipaddr", requestGetRemoteAddress(r)) + rec.Write("size", strconv.FormatInt(size, 10)) + durMicro := int64(dur / time.Microsecond) + rec.Write("durmicro", strconv.FormatInt(durMicro, 10)) + + // to minimize logging, we don't log headers if this is + // self-referal + skipLoggingHeaders := func() bool { + ref := r.Header.Get("Referer") + if ref == "" { + return false + } + return strings.Contains(ref, r.Host) + } + + if !skipLoggingHeaders() { + for k, v := range r.Header { + if !shouldLogHeader(k) { + continue + } + if len(v) > 0 && len(v[0]) > 0 { + rec.Write(k, v[0]) + } + } + } + + _, err := httpLogSiser.WriteRecord(rec) + return err +} + +// requestGetRemoteAddress returns ip address of the client making the request, +// taking into account http proxies +func requestGetRemoteAddress(r *http.Request) string { + hdr := r.Header + hdrRealIP := hdr.Get("x-real-ip") + hdrForwardedFor := hdr.Get("x-forwarded-for") + // Request.RemoteAddress contains port, which we want to remove i.e.: + // "[::1]:58292" => "[::1]" + ipAddrFromRemoteAddr := func(s string) string { + idx := strings.LastIndex(s, ":") + if idx == -1 { + return s + } + return s[:idx] + } + if hdrRealIP == "" && hdrForwardedFor == "" { + return ipAddrFromRemoteAddr(r.RemoteAddr) + } + if hdrForwardedFor != "" { + // X-Forwarded-For is potentially a list of addresses separated with "," + parts := strings.Split(hdrForwardedFor, ",") + for i, p := range parts { + parts[i] = strings.TrimSpace(p) + } + // TODO: should return first non-local address + return parts[0] + } + return hdrRealIP +} + +func hasSpacesCreds() bool { + return os.Getenv("SPACES_KEY") != "" && os.Getenv("SPACES_SECRET") != "" +} + +func newMinioSpacesClient() *MinioClient { + bucket := "kjklogs" + key := os.Getenv("SPACES_KEY") + secret := os.Getenv("SPACES_SECRET") + mc, err := minio.New("nyc3.digitaloceanspaces.com", &minio.Options{ + Creds: credentials.NewStaticV4(key, secret, ""), + Secure: true, + }) + must(err) + found, err := mc.BucketExists(ctx(), bucket) + must(err) + panicIf(!found, "bucket '%s' doesn't exist", bucket) + return &MinioClient{ + c: mc, + bucket: bucket, + } +} + +func minioUploadFilePublic(mc *MinioClient, remotePath string, path string) error { + contentType := mimeTypeFromFileName(remotePath) + opts := minio.PutObjectOptions{ + ContentType: contentType, + } + minioSetPublicObjectMetadata(&opts) + _, err := mc.c.FPutObject(ctx(), mc.bucket, remotePath, path, opts) + return err +} + +func minioSetPublicObjectMetadata(opts *minio.PutObjectOptions) { + if opts.UserMetadata == nil { + opts.UserMetadata = map[string]string{} + } + opts.UserMetadata["x-amz-acl"] = "public-read" +} + +type MinioClient struct { + c *minio.Client + + bucket string +} + +func (c *MinioClient) URLBase() string { + url := c.c.EndpointURL() + return fmt.Sprintf("https://%s.%s/", c.bucket, url.Host) +} + +// --------------------- utils + +func must(err error) { + if err != nil { + panic(err) + } +} + +func panicIf(cond bool, args ...interface{}) { + if !cond { + return + } + s := "condition failed" + if len(args) > 0 { + s = fmt.Sprintf("%s", args[0]) + if len(args) > 1 { + s = fmt.Sprintf(s, args[1:]...) + } + } + panic(s) +} + +var mimeTypes = map[string]string{ + // not present in mime.TypeByExtension() + ".txt": "text/plain", + ".exe": "application/octet-stream", +} + +func mimeTypeFromFileName(path string) string { + ext := strings.ToLower(filepath.Ext(path)) + ct := mimeTypes[ext] + if ct == "" { + ct = mime.TypeByExtension(ext) + } + if ct == "" { + // if all else fails + ct = "application/octet-stream" + } + return ct +} + +func ctx() context.Context { + return context.Background() +} diff --git a/loghttp/readme.md b/loghttp/readme.md new file mode 100644 index 0000000..e8873c6 --- /dev/null +++ b/loghttp/readme.md @@ -0,0 +1,5 @@ +`github.com/kjk/common/loghttp`logs http requests using `siser` to a file that rotates every hour (using `filerate`). + +Optionally it can upload those hourly logs, compressed with brotli, to s3-compatible storage. + +Then you can write code to analyze the logs.