add ZstdCompress; rename flush => sync; remove logtastic.LoggingEnabled

filerotate/filerotate.go

@@ -159,7 +159,7 @@ func (f *File) reopenIfNeeded() error {
 	return f.open(newPath)
 }
 
-func (f *File) write(d []byte, flush bool) (int64, int, error) {
+func (f *File) write(d []byte, sync bool) (int64, int, error) {
 	err := f.reopenIfNeeded()
 	if err != nil {
 		return 0, 0, err
@@ -172,7 +172,7 @@ func (f *File) write(d []byte, flush bool) (int64, int, error) {
 	if err != nil {
 		return 0, n, err
 	}
-	if flush {
+	if sync {
 		err = f.file.Sync()
 	}
 	return f.lastWritePos, n, err
@@ -187,15 +187,15 @@ func (f *File) Write(d []byte) (int, error) {
 	return n, err
 }
 
-// Write2 writes data to a file, optionally flushes. To enable users to later
+// Write2 writes data to a file, optionally syncs to disk. To enable users to later
 // seek to where the data was written, it returns offset at which the data was
 // written, number of bytes and error.
 // You can get path of the file from f.Path
-func (f *File) Write2(d []byte, flush bool) (int64, int, error) {
+func (f *File) Write2(d []byte, sync bool) (int64, int, error) {
 	f.Lock()
 	defer f.Unlock()
 
-	writtenAtPos, n, err := f.write(d, flush)
+	writtenAtPos, n, err := f.write(d, sync)
 	return writtenAtPos, n, err
 }
 
@@ -206,8 +206,8 @@ func (f *File) Close() error {
 	return f.close(false)
 }
 
-// Flush flushes the file
-func (f *File) Flush() error {
+// Sync commits the current contents of the file to stable storage.
+func (f *File) Sync() error {
 	f.Lock()
 	defer f.Unlock()
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/andybalholm/brotli v1.1.0
 	github.com/carlmjohnson/requests v0.23.5
 	github.com/davecgh/go-spew v1.1.1
+	github.com/klauspost/compress v1.17.7
 	github.com/pmezard/go-difflib v1.0.0
 )
 

go.sum

@@ -4,6 +4,8 @@ github.com/carlmjohnson/requests v0.23.5 h1:NPANcAofwwSuC6SIMwlgmHry2V3pLrSqRiSB
 github.com/carlmjohnson/requests v0.23.5/go.mod h1:zG9P28thdRnN61aD7iECFhH5iGGKX2jIjKQD9kqYH+o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=

logtastic/logtastic.go

@@ -30,9 +30,10 @@ const (
 )
 
 var (
-	LoggingEnabled = false
-	Server         = "127.0.0.1:9327"
-	// logtasticServer = "l.arslexis.io"
+	// if true, will compress logs after closing with zstd level 3
+	// and rename to ${name}.zstd
+	CompressLogs   = false
+	Server         = ""
 	ApiKey         = ""
 	LogDir         = ""
 	FileLogs       *filerotate.File
@@ -84,24 +85,24 @@ func logtasticWorker() {
 }
 
 func Stop() {
-	LoggingEnabled = false
+	Server = ""
 	ch <- op{uri: kPleaseStop}
 	if FileLogs != nil {
 		FileLogs.Close()
 	}
-	// if FileErrors != nil {
-	// 	FileErrors.Close()
-	// }
-	// if FileEvents != nil {
-	// 	FileEvents.Close()
-	// }
-	// if FileHits != nil {
-	// 	FileHits.Close()
-	// }
+	if FileErrors != nil {
+		FileErrors.Close()
+	}
+	if FileEvents != nil {
+		FileEvents.Close()
+	}
+	if FileHits != nil {
+		FileHits.Close()
+	}
 }
 
 func logtasticPOST(uriPath string, d []byte, mime string) {
-	if !LoggingEnabled || Server == "" {
+	if Server == "" {
 		return
 	}
 	startLogWorker.Do(func() {

siserlogger/siserlogger.go

@@ -1,12 +1,14 @@
 package siserlogger
 
 import (
+	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
 	"github.com/kjk/common/filerotate"
 	"github.com/kjk/common/siser"
+	"github.com/kjk/common/u"
 )
 
 type File struct {
@@ -32,7 +34,16 @@ func NewDaily(dir string, name string, didRotateFn func(path string)) (*File, er
 	}
 
 	didRotateInternal := func(path string, didRotate bool) {
-		if didRotate && didRotateFn != nil {
+		if !didRotate {
+			return
+		}
+		dst := path + ".zstd"
+		err := u.ZstdCompressFile(dst, path)
+		if err == nil {
+			os.Remove(path)
+			path = dst
+		}
+		if didRotateFn != nil {
 			didRotateFn(path)
 		}
 	}
@@ -53,7 +64,7 @@ func (f *File) Write(d []byte) error {
 	}
 	_, err := f.siser.Write(d, time.Now(), f.RecName)
 	if err == nil {
-		err = f.file.Flush()
+		err = f.file.Sync()
 	}
 	return err
 }

u/compress.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/andybalholm/brotli"
+	"github.com/klauspost/compress/zstd"
 )
 
 // implement io.ReadCloser over os.File wrapped with io.Reader.
@@ -352,3 +353,43 @@ func BrCompressFile(path string) error {
 	}
 	return os.WriteFile(path+".br", d2, 0644)
 }
+
+func ZstdCompressFile(dst string, src string) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	zw, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	// in my tests:
+	// - zstd.SpeedBestCompression is much slower and not much better
+	// - higher concurrency is slower than 2, concurrency 1 produces
+	// much larger files (seems like a bug)
+	w, err := zstd.NewWriter(zw, zstd.WithEncoderLevel(zstd.SpeedBetterCompression), zstd.WithEncoderConcurrency(2))
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, f)
+	if err != nil {
+		zw.Close()
+		os.Remove(dst)
+		return err
+	}
+
+	err = w.Close()
+	if err != nil {
+		zw.Close()
+		os.Remove(dst)
+		return err
+	}
+
+	err = zw.Close()
+	if err != nil {
+		os.Remove(dst)
+		return err
+	}
+	return nil
+}