stat: add mem stat for jemalloc/os

1, add opts arg for mallctl()
2, collect jemalloc.mem info
3, collect os.mem info by /proc
4, add new debug_console cmd:  jmem, osmem, i, q
5, make mem/cmem/stat cmd more human-readable

lualib-src/lua-memory.c

@@ -23,11 +23,41 @@ lblock(lua_State *L) {
 
 static int
 ldumpinfo(lua_State *L) {
-	memory_info_dump();
+	const char *opts = NULL;
+	if (lua_isstring(L, 1)) {
+		opts = luaL_checkstring(L,1);
+	}
+	memory_info_dump(opts);
 
 	return 0;
 }
 
+static int
+ljestat(lua_State *L) {
+	static const char* names[] = {
+		"stats.allocated",
+		"stats.resident",
+		"stats.retained",
+		"stats.mapped",
+		"stats.active" };
+	mallctl_int64("epoch", 1); //refresh je.stats.cache
+	lua_newtable(L);
+	int i;
+	for (i = 0; i < (sizeof(names)/sizeof(names[0])); i++) {
+		lua_pushstring(L, names[i]);
+		lua_pushinteger(L,  (lua_Integer) mallctl_int64(names[i], NULL));
+		lua_settable(L, -3);
+	}
+	return 1;
+}
+
+static int
+lmallctl(lua_State *L) {
+	const char *name = luaL_checkstring(L,1);
+	lua_pushinteger(L, (lua_Integer) mallctl_int64(name, NULL));
+	return 1;
+}
+
 static int
 ldump(lua_State *L) {
 	dump_c_mem();
@@ -69,6 +99,8 @@ luaopen_skynet_memory(lua_State *L) {
 		{ "total", ltotal },
 		{ "block", lblock },
 		{ "dumpinfo", ldumpinfo },
+		{ "jestat", ljestat },
+		{ "mallctl", lmallctl },
 		{ "dump", ldump },
 		{ "info", dump_mem_lua },
 		{ "current", lcurrent },

lualib/skynet/debug.lua

@@ -29,6 +29,7 @@ local function init(skynet, export)
 			stat.mqlen = skynet.stat "mqlen"
 			stat.cpu = skynet.stat "cpu"
 			stat.message = skynet.stat "message"
+			stat.mem = collectgarbage "count"
 			skynet.ret(skynet.pack(stat))
 		end
 

service/debug_console.lua

@@ -55,11 +55,17 @@ local function split_cmdline(cmdline)
 	return split
 end
 
+local function htime()
+	return skynet.hpc() / 1000000000
+end
+
 local function docmd(cmdline, print, fd)
 	local split = split_cmdline(cmdline)
 	local command = split[1]
 	local cmd = COMMAND[command]
 	local ok, list
+	local t_start = htime()
+	print(string.format("<CMD: %s>\n---------------------------------", command))
 	if cmd then
 		ok, list = pcall(cmd, table.unpack(split,2))
 	else
@@ -73,6 +79,7 @@ local function docmd(cmdline, print, fd)
 		end
 	end
 
+	local t_cost = htime() - t_start
 	if ok then
 		if list then
 			if type(list) == "string" then
@@ -81,10 +88,10 @@ local function docmd(cmdline, print, fd)
 				dump_list(print, list)
 			end
 		end
-		print("<CMD OK>")
+		print(string.format("<CMD OK> cost=%.4f sec", t_cost))
 	else
 		print(list)
-		print("<CMD Error>")
+		print(string.format("<CMD Error> cost=%.4f sec", t_cost))
 	end
 end
 
@@ -94,7 +101,7 @@ local function console_main_loop(stdin, print)
 	local ok, err = pcall(function()
 		while true do
 			local cmdline = socket.readline(stdin, "\n")
-			if not cmdline then
+			if not cmdline or cmdline == "q" or cmdline == "q\r" then
 				break
 			end
 			if cmdline:sub(1,4) == "GET " then
@@ -156,12 +163,16 @@ function COMMAND.help()
 		debug = "debug address : debug a lua service",
 		signal = "signal address sig",
 		cmem = "Show C memory info",
+		jmem = "Show jemalloc stats",
+		osmem = "Show OS memory stats like ps: vsz/rss",
 		ping = "ping address",
 		call = "call address ...",
 		trace = "trace address [proto] [on|off]",
 		netstat = "netstat : show netstat",
 		profactive = "profactive [on|off] : active/deactive jemalloc heap profilling",
 		dumpheap = "dumpheap : dump heap profilling",
+		i = "info of this server",
+		q = "close console connection, bye",
 	}
 end
 
@@ -334,14 +345,48 @@ function COMMAND.cmem()
 	local info = memory.info()
 	local tmp = {}
 	for k,v in pairs(info) do
-		tmp[skynet.address(k)] = v
+		tmp[skynet.address(k)] = string.format("%11d  %8.2f Mb", v, v/1048576)
 	end
-	tmp.total = memory.total()
+	local total = memory.total()
+	tmp.total = string.format("%d  %.2f Mb", total, total/1048576)
 	tmp.block = memory.block()
 
 	return tmp
 end
 
+function COMMAND.jmem()
+	local info = memory.jestat()
+	local tmp = {}
+	for k,v in pairs(info) do
+		tmp[k] = string.format("%11d  %8.2f Mb", v, v/1048576)
+	end
+	return tmp
+end
+
+function COMMAND.osmem()
+    local info = {}
+    local fproc = io.open("/proc/self/statm")
+    if fproc then
+        info['vsz'],info['rss'],info['shr'],info['txt'],info['dat'] = string.match(
+                    fproc:read("a"),
+                    '(%d+)%s+(%d+)%s+(%d+)%s+(%d+)%s+%d+%s+(%d+)%s+%d+')
+        if info['vsz'] then
+            for k,v in pairs(info) do
+                info[k] = string.format("%8d Kb  %5d Mb", v*4, math.ceil(v*4/1024))
+            end
+        end
+    end
+    return info
+end
+
+function COMMAND.i()
+    local info = {}
+	info.harbor = skynet.getenv("harbor")
+	info.address = skynet.getenv("address")
+	info.path = os.getenv("PWD")
+    return info
+end
+
 function COMMAND.ping(address)
 	address = adjust_address(address)
 	local ti = skynet.now()

service/launcher.lua

@@ -25,10 +25,14 @@ end
 function command.STAT()
 	local list = {}
 	for k,v in pairs(services) do
+		local t0 = skynet.time()
 		local ok, stat = pcall(skynet.call,k,"debug","STAT")
 		if not ok then
-			stat = string.format("ERROR (%s)",v)
+			stat = {err=stat or "ERROR"}
 		end
+		stat.name = v
+		stat.lag = skynet.time() - t0
+		stat.mem = string.format("%.3f", stat.mem)
 		list[skynet.address(k)] = stat
 	end
 	return list
@@ -44,14 +48,17 @@ end
 
 function command.MEM()
 	local list = {}
+	local sum = 0
 	for k,v in pairs(services) do
 		local ok, kb = pcall(skynet.call,k,"debug","MEM")
 		if not ok then
 			list[skynet.address(k)] = string.format("ERROR (%s)",v)
 		else
-			list[skynet.address(k)] = string.format("%.2f Kb (%s)",kb,v)
+			list[skynet.address(k)] = string.format("%11.2f  %8.2f Mb  (%s)", kb, kb/1024, v)
+			sum = sum + kb
 		end
 	end
+	list["sum"] = string.format("%11.2f Kb  %.2f Mb (sum)", sum, sum/1024)
 	return list
 end
 

skynet-src/malloc_hook.c

@@ -127,8 +127,8 @@ static void malloc_oom(size_t size) {
 }
 
 void
-memory_info_dump(void) {
-	je_malloc_stats_print(0,0,0);
+memory_info_dump(const char* opts) {
+	je_malloc_stats_print(0,0, opts);
 }
 
 bool
@@ -241,7 +241,7 @@ skynet_posix_memalign(void **memptr, size_t alignment, size_t size) {
 #define raw_free free
 
 void
-memory_info_dump(void) {
+memory_info_dump(const char* opts) {
 	skynet_error(NULL, "No jemalloc");
 }
 

skynet-src/malloc_hook.h

@@ -7,7 +7,7 @@
 
 extern size_t malloc_used_memory(void);
 extern size_t malloc_memory_block(void);
-extern void   memory_info_dump(void);
+extern void   memory_info_dump(const char *opts);
 extern size_t mallctl_int64(const char* name, size_t* newval);
 extern int    mallctl_opt(const char* name, int* newval);
 extern bool   mallctl_bool(const char* name, bool* newval);