{"scripts":{"bpftrace/capable":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/capable.bt\n\nprogram = \"\"\"\n// capable.bt Trace security capabilitiy checks (cap_capable()).\n// For Linux, uses eBPF.\n//\n// Copyright 2018 Netflix, Inc.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\nBEGIN\n{\n @cap[0] = \"CAP_CHOWN\";\n @cap[1] = \"CAP_DAC_OVERRIDE\";\n @cap[2] = \"CAP_DAC_READ_SEARCH\";\n @cap[3] = \"CAP_FOWNER\";\n @cap[4] = \"CAP_FSETID\";\n @cap[5] = \"CAP_KILL\";\n @cap[6] = \"CAP_SETGID\";\n @cap[7] = \"CAP_SETUID\";\n @cap[8] = \"CAP_SETPCAP\";\n @cap[9] = \"CAP_LINUX_IMMUTABLE\";\n @cap[10] = \"CAP_NET_BIND_SERVICE\";\n @cap[11] = \"CAP_NET_BROADCAST\";\n @cap[12] = \"CAP_NET_ADMIN\";\n @cap[13] = \"CAP_NET_RAW\";\n @cap[14] = \"CAP_IPC_LOCK\";\n @cap[15] = \"CAP_IPC_OWNER\";\n @cap[16] = \"CAP_SYS_MODULE\";\n @cap[17] = \"CAP_SYS_RAWIO\";\n @cap[18] = \"CAP_SYS_CHROOT\";\n @cap[19] = \"CAP_SYS_PTRACE\";\n @cap[20] = \"CAP_SYS_PACCT\";\n @cap[21] = \"CAP_SYS_ADMIN\";\n @cap[22] = \"CAP_SYS_BOOT\";\n @cap[23] = \"CAP_SYS_NICE\";\n @cap[24] = \"CAP_SYS_RESOURCE\";\n @cap[25] = \"CAP_SYS_TIME\";\n @cap[26] = \"CAP_SYS_TTY_CONFIG\";\n @cap[27] = \"CAP_MKNOD\";\n @cap[28] = \"CAP_LEASE\";\n @cap[29] = \"CAP_AUDIT_WRITE\";\n @cap[30] = \"CAP_AUDIT_CONTROL\";\n @cap[31] = \"CAP_SETFCAP\";\n @cap[32] = \"CAP_MAC_OVERRIDE\";\n @cap[33] = \"CAP_MAC_ADMIN\";\n @cap[34] = \"CAP_SYSLOG\";\n @cap[35] = \"CAP_WAKE_ALARM\";\n @cap[36] = \"CAP_BLOCK_SUSPEND\";\n @cap[37] = \"CAP_AUDIT_READ\";\n @cap[38] = \"CAP_PERFMON\";\n @cap[39] = \"CAP_BPF\";\n @cap[40] = \"CAP_CHECKPOINT_RESTORE\";\n}\n\nkprobe:cap_capable\n{\n $cap = arg2;\n $audit = arg3;\n printf(\\\"time_:%llu uid:%d pid:%d comm:%s cap:%d name:%s audit:%d\\\",\n nsecs, uid, pid, comm, $cap, @cap[$cap], $audit);\n}\n\"\"\"\n\n\ndef cap_capable_func():\n table_name = 'cap_capable_table'\n pxtrace.UpsertTracepoint('cap_capable_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n df = px.DataFrame(table=table_name)\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"cap_capable_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Capable","LongDoc":"Traces calls to the kernel cap_capable() function, which does security capability checks.","orgID":"","hidden":false},"bpftrace/dc_snoop":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/dcsnoop.bt\n\nprogram = \"\"\"\n// dcsnoop Trace directory entry cache (dcache) lookups.\n// For Linux, uses eBPF.\n//\n// Copyright 2018 Netflix, Inc.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\n#include \u003clinux/fs.h\u003e\n\n// from fs/namei.c:\nstruct nameidata {\n struct path path;\n struct qstr last;\n // [...]\n};\n\n// comment out this block to avoid showing hits:\nkprobe:lookup_fast,\nkprobe:lookup_fast.constprop.*\n{\n $nd = (struct nameidata *)arg0;\n printf(\\\"time_:%llu pid:%d comm:%s t:%s file:%s\\\",\n nsecs, pid, comm, \\\"R\\\", str($nd-\u003elast.name));\n}\n\nkprobe:d_lookup\n{\n $name = (struct qstr *)arg1;\n @fname[tid] = $name-\u003ename;\n}\n\nkretprobe:d_lookup\n/@fname[tid]/\n{\n printf(\\\"time_:%llu pid:%d comm:%s t:%s file:%s\\\",\n nsecs, pid, comm, \\\"M\\\", str(@fname[tid]));\n delete(@fname[tid]);\n}\n\"\"\"\n\n\ndef dc_snoop_func():\n table_name = 'dc_snoop_table'\n pxtrace.UpsertTracepoint('dc_snoop_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n df = px.DataFrame(table=table_name)\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"dc_snoop_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"DC Snoop","LongDoc":"Traces directory entry cache (dcache) lookups.","orgID":"","hidden":false},"bpftrace/exec_snoop":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\nprogram = \"\"\"\n#include \u003clinux/sched.h\u003e\n\ntracepoint:syscalls:sys_enter_exec*\n{\n printf(\"time_:%u pid:%d pid_start_time:%d ppid:%d ppid_start_time:%d filename:%s argv0:%s argv1:%s\",\n nsecs,\n pid,\n ((struct task_struct*)curtask)-\u003egroup_leader-\u003estart_time / 10000000,\n ((struct task_struct*)curtask)-\u003eparent-\u003epid,\n ((struct task_struct*)curtask)-\u003eparent-\u003egroup_leader-\u003estart_time / 10000000,\n str(args-\u003efilename), str(args-\u003eargv[0]), str(args-\u003eargv[1]));\n}\n\"\"\"\n\n\ndef add_pod_name(df):\n proc = px.DataFrame(table='process_stats')\n proc.pod = proc.ctx['pod']\n proc.pid = px.upid_to_pid(proc.upid)\n proc.asid = px.upid_to_asid(proc.upid)\n proc = proc.groupby(['pod', 'pid', 'asid']).agg()\n\n # Resolve the pod name of the PID.\n df = df.merge(proc,\n how='left',\n left_on=['pid', 'asid'],\n right_on=['pid', 'asid'],\n suffixes=['', '_x'])\n\n df = df.drop(['pid_x', 'asid_x'])\n\n # Resolve the pod name of the parent PID (PPID).\n df = df.merge(proc,\n how='left',\n left_on=['ppid', 'asid'],\n right_on=['pid', 'asid'],\n suffixes=['', '_x'])\n\n df.parent_pod = df.pod_x\n df = df.drop(['pid_x', 'asid_x'])\n\n return df\n\n\ndef exec_snoop_func(command_substr: str):\n table_name = 'exec_table'\n pxtrace.UpsertTracepoint('exec_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"5m\")\n df = px.DataFrame(table=table_name)\n\n # Grab the node from exec_hostname instead of `ctx`, because not all PIDs have a pod.\n df.node = px._exec_hostname()\n\n # Filter for the requested commands.\n df = df[px.contains(df.filename, command_substr)]\n\n # Try to add the pod name, if available. If pod is not matched, the field will be blank.\n df.asid = px.asid()\n df = add_pod_name(df)\n\n return df['time_', 'node', 'pid', 'pod', 'ppid', 'parent_pod', 'filename', 'argv0', 'argv1']\n","vis":" {\n \"variables\": [\n {\n \"name\": \"command_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The full/partial name of the command on which to filter.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"exec_snoop_func\",\n \"args\": [\n {\n \"name\": \"command_substr\",\n \"variable\": \"command_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Exec Snoop","LongDoc":"Tracing system exec events.","orgID":"","hidden":false},"bpftrace/md_flush":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/mdflush.bt\n\nprogram = \"\"\"\n// mdflush.bt Trace md flush events.\n// For Linux, uses eBPF.\n//\n// Copyright 2018 Netflix, Inc.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\n#include \u003clinux/genhd.h\u003e\n#include \u003clinux/bio.h\u003e\n\nkprobe:md_flush_request\n{\n printf(\\\"time_:%llu pid:%d comm:%s device:%s\\\",\n nsecs, pid, comm, ((struct bio *)arg1)-\u003ebi_disk-\u003edisk_name);\n}\n\"\"\"\n\n\ndef md_flush_func():\n table_name = 'md_flush_table'\n pxtrace.UpsertTracepoint('md_flush_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n df = px.DataFrame(table=table_name)\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"md_flush_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"MD Flush","LongDoc":"Traces flushes at the md driver level, and prints details.","orgID":"","hidden":false},"bpftrace/nap_time":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/naptime.bt\n\nprogram = \"\"\"\n// naptime.bt - Show voluntary sleep calls.\n//\n// Copyright (c) 2019 Brendan Gregg.\n// Licensed under the Apache License, Version 2.0 (the \"License\").\n// This was originally created for the BPF Performance Tools book\n// published by Addison Wesley. ISBN-13: 9780136554820\n// When copying or porting, include this comment.\n\n#include \u003clinux/time.h\u003e\n#include \u003clinux/sched.h\u003e\n\ntracepoint:syscalls:sys_enter_nanosleep\n{\n $task = (struct task_struct *)curtask;\n printf(\\\"time_:%llu ppid:%d pcomm:%s pid:%d comm:%s\\\",\n nsecs, $task-\u003ereal_parent-\u003epid, $task-\u003ereal_parent-\u003ecomm, pid, comm);\n}\n\"\"\"\n\n\ndef nap_time_func():\n table_name = 'nap_time_table'\n pxtrace.UpsertTracepoint('nap_time_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n df = px.DataFrame(table=table_name)\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"nap_time_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"NAP Time","LongDoc":"Tracing application sleeps via the nanosleep(2) syscall.","orgID":"","hidden":false},"bpftrace/oom_kill":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/oomkill.bt\n\nprogram = \"\"\"\n// oomkill.bt Trace OOM killer.\n// For Linux, uses eBPF.\n//\n// Copyright 2018 Netflix, Inc.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\n#include \u003clinux/oom.h\u003e\n\nkprobe:oom_kill_process\n{\n $oc = (struct oom_control *)arg0;\n printf(\\\"time_:%llu triggered_by_pid:%d triggered_by_comm:%s killed_pid:%d killed_comm:%s pages:%d\\\",\n nsecs, pid, comm, $oc-\u003echosen-\u003epid, $oc-\u003echosen-\u003ecomm, $oc-\u003etotalpages);\n}\n\"\"\"\n\n\ndef oom_kill_func():\n table_name = 'oom_kill_table'\n pxtrace.UpsertTracepoint('oom_kill_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n df = px.DataFrame(table=table_name)\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"oom_kill_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"OOM Killer","LongDoc":"Traces the Linux out-of-memory (OOM) killer and shows basic details on one line per OOM kill.","orgID":"","hidden":false},"bpftrace/socket_size":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n# WARNING: Depending on the size of your cluster and how network-intensive your Pods and services are, this tool\n# can generate a high volume of profiling data because it traces every single socket send/receive call.\n# Therefore, run it judiciously and for an approperiate amount of time based on your needs.\n\nimport pxtrace\nimport px\n\nsendmsg_program = \"\"\"\n// Adapted From \"BPF Performance Tools\" book by Brendan Gregg; Chapter 10: Networking\n// Available at: https://github.com/brendangregg/bpf-perf-tools-book/blob/master/originals/Ch10_Networking/socksize.bt\n\n/*\n * socket_sendmsg tool shows the socket stats (count, size, and throughput) for write socket I/O requests.\n *\n * See BPF Performance Tools, Chapter 10, for an explanation of this tool.\n *\n * Copyright (c) 2019 Brendan Gregg.\n * Licensed under the Apache License, Version 2.0 (the \"License\").\n * This was originally created for the BPF Performance Tools book\n * published by Addison Wesley. ISBN-13: 9780136554820\n * When copying or porting, include this comment.\n *\n * 12-Apr-2019 Brendan Gregg Created this.\n * 5-May-2023 Ata Fatahi modified to extract more info such as source and destination IP:Port pairs.\n */\n\n#include \u003clinux/socket.h\u003e\n#include \u003cnet/sock.h\u003e\n\n\nkprobe:sock_sendmsg\n{\n $AF_INET = (uint16) 2;\n\n $sock = (struct socket *)arg0;\n @ssocket[tid] = $sock;\n $s = (struct sock *) $sock-\u003esk;\n $inet_family = $s-\u003e__sk_common.skc_family;\n\n if ($inet_family == $AF_INET) {\n $daddr = ntop($s-\u003e__sk_common.skc_daddr);\n $saddr = ntop($s-\u003e__sk_common.skc_rcv_saddr);\n } else {\n $daddr = ntop($s-\u003e__sk_common.skc_v6_daddr.in6_u.u6_addr8);\n $saddr = ntop($s-\u003e__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8);\n }\n\n $lport = $s-\u003e__sk_common.skc_num;\n $dport = $s-\u003e__sk_common.skc_dport;\n\n @ssock[tid] = $s;\n @sdad[tid]= $daddr;\n @sdp[tid]= $dport;\n @ssad[tid] = $saddr;\n @slp[tid] = $lport;\n}\n\nkretprobe:sock_sendmsg\n{\n printf(\\\"time_:%llu tid:%u ssock:%d src_ip:%s lport:%d dst_ip:%s dst_port:%d size:%d\",\n nsecs,\n tid,\n @ssock[tid],\n @ssad[tid],\n @slp[tid],\n @sdad[tid],\n @sdp[tid],\n retval);\n\n delete(@ssocket[tid]);\n delete(@ssock[tid]);\n delete(@sdad[tid]);\n delete(@sdp[tid]);\n delete(@ssad[tid]);\n delete(@slp[tid]);\n}\n\n\"\"\"\n\nrecvmsg_program = \"\"\"\n// Adapted From \"BPF Performance Tools\" book by Brendan Gregg; Chapter 10: Networking\n// Available at: https://github.com/brendangregg/bpf-perf-tools-book/blob/master/originals/Ch10_Networking/socksize.bt\n\n/*\n * socket_recvmsg tool shows the socket stats (count, size, and throughput) for read socket I/O requests.\n *\n * See BPF Performance Tools, Chapter 10, for an explanation of this tool.\n *\n * Copyright (c) 2019 Brendan Gregg.\n * Licensed under the Apache License, Version 2.0 (the \"License\").\n * This was originally created for the BPF Performance Tools book\n * published by Addison Wesley. ISBN-13: 9780136554820\n * When copying or porting, include this comment.\n *\n * 12-Apr-2019 Brendan Gregg Created this.\n * 5-May-2023 Ata Fatahi modified to extract more info such as source and destination IP:Port pairs.\n */\n\n#include \u003clinux/socket.h\u003e\n#include \u003cnet/sock.h\u003e\n\n\nkprobe:sock_recvmsg\n{\n $AF_INET = (uint16) 2;\n\n $sock = (struct socket *)arg0;\n @rsocket[tid] = $sock;\n $s = (struct sock *) $sock-\u003esk;\n $inet_family = $s-\u003e__sk_common.skc_family;\n\n if ($inet_family == $AF_INET) {\n $daddr = ntop($s-\u003e__sk_common.skc_daddr);\n $saddr = ntop($s-\u003e__sk_common.skc_rcv_saddr);\n } else{\n $daddr = ntop($s-\u003e__sk_common.skc_v6_daddr.in6_u.u6_addr8);\n $saddr = ntop($s-\u003e__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8);\n }\n\n $lport = $s-\u003e__sk_common.skc_num;\n $dport = $s-\u003e__sk_common.skc_dport;\n\n @rsock[tid] = $s;\n @rdad[tid]= $daddr;\n @rdp[tid]= $dport;\n @rsad[tid] = $saddr;\n @rlp[tid] = $lport;\n}\n\nkretprobe:sock_recvmsg\n{\n printf(\\\"time_:%llu tid:%u rsock:%d src_ip:%s lport:%d dst_ip:%s dst_port:%d size:%d\",\n nsecs,\n tid,\n @rsock[tid],\n @rsad[tid],\n @rlp[tid],\n @rdad[tid],\n @rdp[tid],\n retval);\n\n delete(@rsocket[tid]);\n delete(@rsock[tid]);\n delete(@rdad[tid]);\n delete(@rdp[tid]);\n delete(@rsad[tid]);\n delete(@rlp[tid]);\n}\n\n\"\"\"\n\n\ndef sock_sendmsg_func():\n\n duration = \"1m\"\n table_name = 'sock_sendmsg_table'\n pxtrace.UpsertTracepoint('sock_sendmsg_tracer',\n table_name,\n sendmsg_program,\n pxtrace.kprobe(),\n duration)\n\n df = px.DataFrame(table=table_name)\n\n # Convert IPs to domain names.\n df.src = px.pod_id_to_pod_name(px.ip_to_pod_id(df.src_ip))\n df.src = px.select(df.src == '', df.src_ip, df.src)\n df.dst = px.pod_id_to_pod_name(px.ip_to_pod_id(df.dst_ip))\n df.dst = px.select(df.dst == '', df.dst_ip, df.dst)\n\n df = df.head(100000)\n\n df = df.groupby(['src', 'dst']).agg(count=('size', px.count), total_size=('size', px.sum))\n df['avg_size'] = df['total_size'] / df['count']\n df['throughput'] = df['total_size'] / (px.atoi(px.substring(duration, 0, px.length(duration) - 1), 1) * 60)\n\n # Filter for a particular service/pod, if desired.\n df = df[px.contains(df['src'], '')]\n\n return df\n\n\ndef sock_recvmsg_func():\n\n duration = \"1m\"\n table_name = 'sock_recvmsg_table'\n pxtrace.UpsertTracepoint('sock_recvmsg_tracer',\n table_name,\n recvmsg_program,\n pxtrace.kprobe(),\n duration)\n\n df = px.DataFrame(table=table_name)\n\n # Convert IPs to domain names.\n df.src = px.pod_id_to_pod_name(px.ip_to_pod_id(df.src_ip))\n df.src = px.select(df.src == '', df.src_ip, df.src)\n df.dst = px.pod_id_to_pod_name(px.ip_to_pod_id(df.dst_ip))\n df.dst = px.select(df.dst == '', df.dst_ip, df.dst)\n\n df = df.head(100000)\n\n df = df.groupby(['src', 'dst']).agg(count=('size', px.count), total_size=('size', px.sum))\n df['avg_size'] = df['total_size'] / df['count']\n df['throughput'] = df['total_size'] / (px.atoi(px.substring(duration, 0, px.length(duration) - 1), 1) * 60)\n\n # Filter for a particular service/pod, if desired.\n df = df[px.contains(df['src'], '')]\n\n return df\n","vis":"","placement":"","ShortDoc":"Socket Size","LongDoc":"Shows info and size stats for socket I/O requests.","orgID":"","hidden":false},"bpftrace/sync_snoop":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/syncsnoop.bt\n\nprogram = \"\"\"\n// syncsnoop.bt Trace sync() variety of syscalls.\n// For Linux, uses eBPF.\n//\n// Copyright 2018 Netflix, Inc.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\ntracepoint:syscalls:sys_enter_sync,\ntracepoint:syscalls:sys_enter_syncfs,\ntracepoint:syscalls:sys_enter_fsync,\ntracepoint:syscalls:sys_enter_fdatasync,\ntracepoint:syscalls:sys_enter_sync_file_range*,\ntracepoint:syscalls:sys_enter_msync\n{\n printf(\\\"time_:%llu pid:%-6d comm:%-16s event:%s\\\",\n nsecs, pid, comm, probe);\n}\n\"\"\"\n\n\ndef sync_snoop_func():\n table_name = 'sync_snoop_table'\n pxtrace.UpsertTracepoint('sync_snoop_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n df = px.DataFrame(table=table_name)\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"sync_snoop_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Sync Snoop","LongDoc":"Tracing file system sync events.","orgID":"","hidden":false},"bpftrace/tcp_drops":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n# flake8: noqa:E501\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpdrop.bt\n# Due to backported changes (i.e. the kprobe:tcp_drop was removed in older versions of the kernel),\n# the old bpftrace script may not work on some older kernels \u003c5.19.\npre_519_program = pxtrace.TraceProgram(\"\"\"\n// tcpdrop.bt Trace TCP kernel-dropped packets/segments.\n// For Linux, uses bpftrace and eBPF.\n//\n// Copyright (c) 2018 Dale Hamel.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\n#include \u003clinux/socket.h\u003e\n#include \u003cnet/sock.h\u003e\n\nBEGIN\n{\n // See https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h\n @tcp_states[1] = \"ESTABLISHED\";\n @tcp_states[2] = \"SYN_SENT\";\n @tcp_states[3] = \"SYN_RECV\";\n @tcp_states[4] = \"FIN_WAIT1\";\n @tcp_states[5] = \"FIN_WAIT2\";\n @tcp_states[6] = \"TIME_WAIT\";\n @tcp_states[7] = \"CLOSE\";\n @tcp_states[8] = \"CLOSE_WAIT\";\n @tcp_states[9] = \"LAST_ACK\";\n @tcp_states[10] = \"LISTEN\";\n @tcp_states[11] = \"CLOSING\";\n @tcp_states[12] = \"NEW_SYN_RECV\";\n}\n\nkprobe:tcp_drop\n{\n $sk = ((struct sock *) arg0);\n $inet_family = $sk-\u003e__sk_common.skc_family;\n\n $AF_INET = (uint16) 2;\n $AF_INET6 = (uint16) 10;\n\n if ($inet_family == $AF_INET || $inet_family == $AF_INET6) {\n if ($inet_family == $AF_INET) {\n $daddr = ntop($sk-\u003e__sk_common.skc_daddr);\n $saddr = ntop($sk-\u003e__sk_common.skc_rcv_saddr);\n } else {\n $daddr = ntop($sk-\u003e__sk_common.skc_v6_daddr.in6_u.u6_addr8);\n $saddr = ntop($sk-\u003e__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8);\n }\n $sport = $sk-\u003e__sk_common.skc_num;\n $dport = $sk-\u003e__sk_common.skc_dport;\n\n // Destination port is big endian, it must be flipped\n $dport = ($dport \u003e\u003e 8) | (($dport \u003c\u003c 8) \u0026 0x00FF00);\n\n $state = $sk-\u003e__sk_common.skc_state;\n $statestr = @tcp_states[$state];\n\n printf(\\\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\\\",\n nsecs,\n pid,\n ((struct task_struct*)curtask)-\u003egroup_leader-\u003estart_time / 10000000,\n $saddr,\n $sport,\n $daddr,\n $dport,\n $statestr);\n }\n}\n\"\"\",\nmax_kernel='5.18'\n)\n\npost_519_program = pxtrace.TraceProgram(\n\"\"\"\n// tcpdrop.bt Trace TCP kernel-dropped packets/segments.\n// For Linux, uses bpftrace and eBPF.\n//\n// Copyright (c) 2018 Dale Hamel.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\n#include \u003clinux/socket.h\u003e\n#include \u003cnet/sock.h\u003e\n\nBEGIN\n{\n // See https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h\n @tcp_states[1] = \"ESTABLISHED\";\n @tcp_states[2] = \"SYN_SENT\";\n @tcp_states[3] = \"SYN_RECV\";\n @tcp_states[4] = \"FIN_WAIT1\";\n @tcp_states[5] = \"FIN_WAIT2\";\n @tcp_states[6] = \"TIME_WAIT\";\n @tcp_states[7] = \"CLOSE\";\n @tcp_states[8] = \"CLOSE_WAIT\";\n @tcp_states[9] = \"LAST_ACK\";\n @tcp_states[10] = \"LISTEN\";\n @tcp_states[11] = \"CLOSING\";\n @tcp_states[12] = \"NEW_SYN_RECV\";\n}\n\ntracepoint:skb:kfree_skb\n{\n $reason = args-\u003ereason;\n $skb = (struct sk_buff *)args-\u003eskbaddr;\n $sk = ((struct sock *) $skb-\u003esk);\n $inet_family = $sk-\u003e__sk_common.skc_family;\n\n if ($reason \u003e SKB_DROP_REASON_NOT_SPECIFIED \u0026\u0026\n ($inet_family == AF_INET || $inet_family == AF_INET6)) {\n if ($inet_family == AF_INET) {\n $daddr = ntop($sk-\u003e__sk_common.skc_daddr);\n $saddr = ntop($sk-\u003e__sk_common.skc_rcv_saddr);\n } else {\n $daddr = ntop($sk-\u003e__sk_common.skc_v6_daddr.in6_u.u6_addr8);\n $saddr = ntop($sk-\u003e__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8);\n }\n $lport = $sk-\u003e__sk_common.skc_num;\n $dport = $sk-\u003e__sk_common.skc_dport;\n\n // Destination port is big endian, it must be flipped\n $dport = bswap($dport);\n\n $state = $sk-\u003e__sk_common.skc_state;\n $statestr = @tcp_states[$state];\n\n printf(\\\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\\\",\n nsecs,\n pid,\n ((struct task_struct*)curtask)-\u003egroup_leader-\u003estart_time / 10000000,\n $saddr,\n $lport,\n $daddr,\n $dport,\n $statestr);\n }\n}\n\"\"\",\nmin_kernel='5.19'\n)\n\n\ndef tcp_drops_func():\n table_name = 'tcp_drop_table'\n pxtrace.UpsertTracepoint('tcp_drop_tracer',\n table_name,\n [pre_519_program, post_519_program],\n pxtrace.kprobe(),\n \"10m\")\n\n df = px.DataFrame(table=table_name)\n\n # Convert IPs to domain names.\n df.src = px.pod_id_to_pod_name(px.ip_to_pod_id(df.src_ip))\n df.src = px.select(df.src == '', df.src_ip, df.src)\n df.dst = px.nslookup(df.dst_ip)\n\n # Count drops.\n df = df.groupby(['src', 'dst']).agg(drops=('src', px.count))\n\n # Filter for a particular service, if desired.\n df = df[px.contains(df['dst'], '')]\n\n # Set a threshold to display, if desired.\n df = df[df['drops'] \u003e 0]\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"tcp_drops_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"TCP Drops\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"src\",\n \"toColumn\": \"dst\"\n },\n \"edgeWeightColumn\": \"drops\",\n \"edgeColorColumn\": \"drops\",\n \"edgeLength\": 300,\n \"edgeThresholds\": {\n \"mediumThreshold\": 5,\n \"highThreshold\": 50\n },\n \"edgeHoverInfo\": [\n \"drops\"\n ]\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"TCP drops","LongDoc":"Shows TCP drop counts in the cluster.","orgID":"","hidden":false},"bpftrace/tcp_retransmits":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n# flake8: noqa:E501\n\nimport pxtrace\nimport px\n\n# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpretrans.bt\nprogram = \"\"\"\n// tcpretrans.bt Trace or count TCP retransmits\n// For Linux, uses bpftrace and eBPF.\n//\n// Copyright (c) 2018 Dale Hamel.\n// Licensed under the Apache License, Version 2.0 (the \"License\")\n\n#include \u003clinux/socket.h\u003e\n#include \u003cnet/sock.h\u003e\n\nBEGIN\n{\n // See include/net/tcp_states.h:\n @tcp_states[1] = \\\"ESTABLISHED\\\";\n @tcp_states[2] = \\\"SYN_SENT\\\";\n @tcp_states[3] = \\\"SYN_RECV\\\";\n @tcp_states[4] = \\\"FIN_WAIT1\\\";\n @tcp_states[5] = \\\"FIN_WAIT2\\\";\n @tcp_states[6] = \\\"TIME_WAIT\\\";\n @tcp_states[7] = \\\"CLOSE\\\";\n @tcp_states[8] = \\\"CLOSE_WAIT\\\";\n @tcp_states[9] = \\\"LAST_ACK\\\";\n @tcp_states[10] = \\\"LISTEN\\\";\n @tcp_states[11] = \\\"CLOSING\\\";\n @tcp_states[12] = \\\"NEW_SYN_RECV\\\";\n}\n\nkprobe:tcp_retransmit_skb\n{\n $sk = (struct sock *)arg0;\n $inet_family = $sk-\u003e__sk_common.skc_family;\n\n $AF_INET = (uint16) 2;\n $AF_INET6 = (uint16) 10;\n\n if ($inet_family == $AF_INET || $inet_family == $AF_INET6) {\n // initialize variable type:\n $daddr = ntop(0);\n $saddr = ntop(0);\n if ($inet_family == $AF_INET) {\n $daddr = ntop($sk-\u003e__sk_common.skc_daddr);\n $saddr = ntop($sk-\u003e__sk_common.skc_rcv_saddr);\n } else {\n $daddr = ntop($sk-\u003e__sk_common.skc_v6_daddr.in6_u.u6_addr8);\n $saddr = ntop($sk-\u003e__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8);\n }\n $lport = $sk-\u003e__sk_common.skc_num;\n $dport = $sk-\u003e__sk_common.skc_dport;\n\n // Destination port is big endian, it must be flipped\n $dport = ($dport \u003e\u003e 8) | (($dport \u003c\u003c 8) \u0026 0x00FF00);\n\n $state = $sk-\u003e__sk_common.skc_state;\n $statestr = @tcp_states[$state];\n\n printf(\\\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\\\",\n nsecs,\n pid,\n ((struct task_struct*)curtask)-\u003egroup_leader-\u003estart_time / 10000000,\n $saddr,\n $lport,\n $daddr,\n $dport,\n $statestr);\n }\n}\n\"\"\"\n\n\ndef tcp_retransmissions_func():\n table_name = 'tcp_retransmissions'\n pxtrace.UpsertTracepoint('tcp_retransmit_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"10m\")\n\n df = px.DataFrame(table=table_name)\n\n # Convert IPs to domain names.\n # Source is expected to be pod, while destination is expected to be a service.\n df.src = px.pod_id_to_pod_name(px.ip_to_pod_id(df.src_ip))\n df.src = px.select(df.src == '', df.src_ip, df.src)\n df.dst = px.nslookup(df.dst_ip)\n\n # Count retransmissions.\n df = df.groupby(['src', 'dst']).agg(retransmissions=('src', px.count))\n\n # Filter for a particular service, if desired.\n df = df[px.contains(df['dst'], '')]\n\n # Set a threshold to display, if desired.\n df = df[df['retransmissions'] \u003e 0]\n\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"tcp_retransmissions_func\",\n \"args\": []\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"TCP Retransmits\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"src\",\n \"toColumn\": \"dst\"\n },\n \"edgeWeightColumn\": \"retransmissions\",\n \"edgeColorColumn\": \"retransmissions\",\n \"edgeLength\": 300,\n \"edgeThresholds\": {\n \"mediumThreshold\": 5,\n \"highThreshold\": 50\n },\n \"edgeHoverInfo\": [\n \"retransmissions\"\n ]\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"TCP retransmissions","LongDoc":"Shows TCP retransmission counts in the cluster.","orgID":"","hidden":false},"px/agent_status":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\npx.display(px.GetAgentStatus())\n","vis":"","placement":"","ShortDoc":"Get agent status.","LongDoc":"This script gets the status of all the pixie agents (PEMs/Collectors) running.\n","orgID":"","hidden":false},"px/agent_status_diagnostics":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n# Exclude kelvins since they don't run BPF programs nor\n# do they have the host linux headers accessible\ndf = px.GetAgentStatus(False)\ndf = df.agg(\n installed=('kernel_headers_installed', px.sum),\n count=('kernel_headers_installed', px.count),\n)\ndf.headers_installed_percent = df.installed / df.count\ndf = df.drop(['installed', 'count'])\npx.display(df)\n","vis":"","placement":"","ShortDoc":"Agent status diagnostics","LongDoc":"This script performs diagnostics on the agents' (PEMs/Collectors) status\n","orgID":"","hidden":false},"px/amqp_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n\"\"\" AMQP Data Tracer\n\nThis script traces all AMQP (Rabbitmq) data on the cluster.\n\"\"\"\nimport px\n\n\ndef amqp_data(start_time: str, source_filter: str, destination_filter: str):\n \"\"\"\n Returns the raw AMQP data\n \"\"\"\n df = px.DataFrame(table=\"amqp_events\", start_time=start_time)\n df = sanitize_filter(df, source_filter, destination_filter)\n df = df[\n \"time_\",\n \"upid\",\n \"frame_name\",\n \"channel\",\n \"req_name\",\n \"resp_name\",\n \"req_msg\",\n \"resp_msg\",\n \"source\",\n \"destination\",\n \"latency\",\n ]\n return df\n\n\ndef amqp_msg_stats(start_time: str, source_filter: str, destination_filter: str):\n \"\"\"\n Returns the number of messages for each AMQP message type\n \"\"\"\n df = px.DataFrame(table=\"amqp_events\", start_time=start_time)\n df = sanitize_filter(df, source_filter, destination_filter)\n df = df[\n \"time_\", \"upid\", \"frame_name\", \"req_name\", \"resp_name\", \"req_msg\", \"resp_msg\"\n ]\n df_stats_per_req = df.groupby([\"frame_name\", \"req_name\"]).agg(\n stats=(\"req_name\", px.count)\n )\n return df_stats_per_req\n\n\ndef amqp_average_stats(start_time: str, source_filter: str, destination_filter: str):\n \"\"\"\n Returns the average AMQP body size\n \"\"\"\n df = px.DataFrame(table=\"amqp_events\", start_time=start_time)\n df = sanitize_filter(df, source_filter, destination_filter)\n df = df[df[\"frame_name\"] == \"Content Header\"]\n df.body_size = px.pluck_int64(df.req_msg, \"body_size\")\n body_size_stats = df.groupby(\"frame_name\").agg(\n avg_body_size=(\"body_size\", px.mean),\n max_body_size=(\"body_size\", px.max),\n min_body_size=(\"body_size\", px.min),\n )\n\n return body_size_stats\n\n\ndef amqp_producer_pod_list(\n start_time: str, source_filter: str, destination_filter: str\n):\n df = px.DataFrame(table=\"amqp_events\", start_time=start_time)\n df = sanitize_filter(df, source_filter, destination_filter)\n df = df[df[\"req_name\"] == \"BasicPublish\"]\n df.routing_key = px.pluck(df.req_msg, \"routing_key\")\n df = df.groupby([\"routing_key\", \"pod\", \"namespace\", \"source\", \"destination\"]).agg(\n count=(\"pod\", px.count)\n )\n df = df[\"routing_key\", \"namespace\", \"pod\", \"source\", \"destination\"]\n return df\n\n\ndef amqp_consumer_pod_list(\n start_time: str, source_filter: str, destination_filter: str\n):\n df = px.DataFrame(table=\"amqp_events\", start_time=start_time)\n df = sanitize_filter(df, source_filter, destination_filter)\n\n df = df[df[\"resp_name\"] == \"BasicDeliver\"][\n \"resp_msg\", \"namespace\", \"pod\", \"source\", \"destination\"\n ]\n df.routing_key = px.pluck(df.resp_msg, \"routing_key\")\n df.consumer_tag = px.pluck(df.resp_msg, \"consumer_tag\")\n\n df = df.groupby(\n [\"routing_key\", \"pod\", \"namespace\", \"source\", \"destination\", \"consumer_tag\"]\n ).agg(count=(\"pod\", px.count))\n\n df = df[\"routing_key\", \"namespace\", \"pod\", \"source\", \"destination\", \"consumer_tag\"]\n return df\n\n\ndef amqp_flow_graph(\n start_time: str, ns: px.Namespace, source_filter: str, destination_filter: str\n):\n \"\"\"\n Adds flow graph from producers to consumers\n \"\"\"\n df = px.DataFrame(table=\"amqp_events\", start_time=start_time)\n\n # Get produce and consumer records\n producer_df = amqp_producer_pod_list(start_time, source_filter, destination_filter)\n consumer_df = amqp_consumer_pod_list(start_time, source_filter, destination_filter)\n\n producer_df = filter_if_not_empty(producer_df, \"namespace\", ns)\n consumer_df = filter_if_not_empty(consumer_df, \"namespace\", ns)\n\n producer_df.src = producer_df.source\n producer_df.dest = (\n producer_df.destination + \"/routing_key/\" + producer_df.routing_key\n )\n\n consumer_df.src = (\n consumer_df.destination + \"/routing_key/\" + consumer_df.routing_key\n )\n consumer_df.dest = consumer_df.source\n\n producer_df = producer_df[\"src\", \"dest\"]\n consumer_df = consumer_df[\"src\", \"dest\"]\n\n df = producer_df.append(consumer_df)\n\n return df\n\n\ndef sanitize_filter(df, source_filter, destination_filter):\n \"\"\"\n Converts class id \u0026 method id to readable req_name \u0026 resp_name\n \"\"\"\n df = add_source_dest_columns(df)\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n df.frame_name = px.amqp_frame_type_name(df.frame_type)\n df.req_name = px.amqp_method_name(df.req_class_id, df.req_method_id)\n df.resp_name = px.amqp_method_name(\n df.resp_class_id, df.resp_method_id\n )\n return df\n\n\ndef add_source_dest_columns(df):\n \"\"\"Add source and destination columns for the AMQP request.\n \"\"\"\n df.pod = df.ctx[\"pod\"]\n df.namespace = df.ctx[\"namespace\"]\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != \"\"\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != \"\"]\n df = df[df.destination != \"\"]\n\n df = df.drop([\"ra_pod\", \"is_ra_pod\", \"ra_name\", \"is_server_tracing\"])\n\n return df\n\n\ndef filter_if_not_empty(df, column: str, val: str):\n \"\"\"\n Filters for rows where column is equal to val. If val is \"\", all rows are retained.\n \"\"\"\n df.criterion = px.select(df[column] == val, True, val == \"\")\n df = df[df.criterion]\n df = df.drop(\"criterion\")\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"routing_key\",\n \"type\": \"PX_STRING\",\n \"description\": \"The routing key to filter on\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"amqp_data\",\n \"func\": {\n \"name\": \"amqp_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"amqp_msg_stats\",\n \"func\": {\n \"name\": \"amqp_msg_stats\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"amqp_average_stats\",\n \"func\": {\n \"name\": \"amqp_average_stats\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"amqp_producer_pod_list\",\n \"func\": {\n \"name\": \"amqp_producer_pod_list\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"amqp_consumer_pod_list\",\n \"func\": {\n \"name\": \"amqp_consumer_pod_list\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"amqp_flow_graph\",\n \"func\": {\n \"name\": \"amqp_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"AMQP Raw Data\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"amqp_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"AMQP Raw Packet stats\",\n \"position\": {\n \"x\": 0,\n \"y\": 4,\n \"w\": 6,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"amqp_msg_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"AMQP AVG Packet stats\",\n \"position\": {\n \"x\": 6,\n \"y\": 4,\n \"w\": 6,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"amqp_average_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"AMQP Producer List\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"amqp_producer_pod_list\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"AMQP Consumer List\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"amqp_consumer_pod_list\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"AMQP Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"amqp_flow_graph\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"src\",\n \"toColumn\": \"dest\"\n },\n \"enableDefaultHierarchy\": true,\n \"edgeLength\": 300\n }\n }\n ]\n}\n","placement":"","ShortDoc":"AMQP messages","LongDoc":"Shows a sample of amqp messages in the cluster.\n","orgID":"","hidden":false},"px/cluster":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Cluster Overview\n\nThis view lists the namespaces and the nodes that are available on the current cluster.\n\n'''\nimport px\nimport pxviews\n\n\ndef nodes_for_cluster(start_time: str):\n ''' Gets a list of nodes in the current cluster since `start_time`.\n Args:\n @start_time Start time of the data to examine.\n '''\n df = pxviews.container_process_summary(px.now() + px.parse_duration(start_time), px.now())\n agg = df.groupby(['node', 'pod']).agg()\n pod_per_node_count = agg.groupby('node').agg(pod_count=('pod', px.count))\n df = df.groupby(['node']).agg(\n cpu_usage=('cpu_usage', px.sum),\n )\n df.cpu_usage = px.Percent(df.cpu_usage)\n output = df.merge(pod_per_node_count, how='right', left_on='node', right_on='node',\n suffixes=['', '_x'])\n return output[['node', 'cpu_usage', 'pod_count']]\n\n\ndef pods_for_cluster(start_time: str):\n ''' A list of pods in the cluster.\n Args:\n @start_time: The timestamp of data to start at.\n '''\n df = pxviews.pod_resource_stats(px.now() + px.parse_duration(start_time), px.now())\n df.start_time = df.pod_start_time\n df.status = df.pod_status\n return df[[\n 'pod', 'cpu_usage', 'total_disk_read_throughput',\n 'total_disk_write_throughput', 'container_count',\n 'node', 'start_time', 'status',\n ]]\n\n\ndef namespaces_for_cluster(start_time: str):\n ''' Gets a overview of namespaces in the current cluster since `start_time`.\n Args:\n @start_time Start time of the data to examine.\n '''\n df = pxviews.container_process_summary(px.now() + px.parse_duration(start_time), px.now())\n agg = df.groupby(['service', 'pod', 'namespace']).agg()\n pod_count = agg.groupby(['namespace', 'pod']).agg()\n pod_count = pod_count.groupby('namespace').agg(pod_count=('pod', px.count))\n svc_count = agg.groupby(['namespace', 'service']).agg()\n svc_count = svc_count.groupby('namespace').agg(service_count=('service', px.count))\n pod_and_svc_count = pod_count.merge(svc_count, how='inner',\n left_on='namespace', right_on='namespace',\n suffixes=['', '_x'])\n df = df.groupby(['namespace']).agg(\n vsize=('vsize', px.sum),\n rss=('rss', px.sum),\n )\n output = df.merge(pod_and_svc_count, how='inner', left_on='namespace',\n right_on='namespace', suffixes=['', '_y'])\n return output[['namespace', 'pod_count', 'service_count', 'vsize', 'rss']]\n\n\ndef services_for_cluster(start_time: str):\n ''' Get an overview of the services in the current cluster.\n Args:\n @start_time: The timestamp of data to start at.\n '''\n pod_count = pxviews.container_process_summary(px.now() + px.parse_duration(start_time), px.now())\n pod_count = pod_count.groupby(['service', 'pod', 'namespace']).agg()\n pod_count = pod_count[pod_count.service != '']\n pod_count = pod_count.groupby('service').agg(pod_count=('pod', px.count))\n\n service_let = service_let_summary(start_time)\n df = pod_count.merge(\n service_let,\n how=\"left\",\n left_on=\"service\",\n right_on=\"service\",\n suffixes=[\"\", \"_x\"],\n )\n return df[['service', 'pod_count', 'http_latency_in', 'http_req_throughput_in', 'http_error_rate_in',\n 'inbound_conns', 'outbound_conns']]\n\n\ndef service_let_summary(start_time: str):\n ''' Compute a summary of traffic by requesting service, for requests\n on services in the current cluster.\n Args:\n @start_time: The timestamp of data to start at.\n '''\n conn_stats_df = pxviews.connection_throughput_stats(start_time, px.now()).drop('time_')\n conn_stats_df.service = conn_stats_df.ctx['service']\n conn_stats_df = conn_stats_df.groupby(['service']).agg(\n inbound_conn_throughput=('inbound_conn_throughput', px.sum),\n outbound_conn_throughput=('outbound_conn_throughput', px.sum),\n )\n\n window = px.DurationNanos(px.now() - (px.now() + px.parse_duration(start_time)))\n conn_stats_df.inbound_conns = conn_stats_df.inbound_conn_throughput / window\n conn_stats_df.outbound_conns = conn_stats_df.outbound_conn_throughput / window\n\n http_stats_df = pxviews.inbound_http_summary(start_time=start_time, end_time=px.now())\n http_stats_df.service = http_stats_df.ctx['service']\n\n http_stats_df = http_stats_df.groupby(['service']).agg(\n http_req_count_in=('num_requests', px.sum),\n http_error_count_in=('num_errors', px.sum),\n # TODO usse a combine_quantiles UDF to merge quantiles\n http_latency_in=('latency_quantiles', px.any),\n )\n\n # Compute throughput values.\n http_stats_df.http_req_throughput_in = http_stats_df.http_req_count_in / window\n http_stats_df.http_error_rate_in = px.Percent(\n px.select(\n http_stats_df.http_req_count_in != 0,\n http_stats_df.http_error_count_in / http_stats_df.http_req_count_in,\n 0.0,\n )\n )\n\n # Merge conn_stats_df and http_stats_df.\n df = conn_stats_df.merge(http_stats_df,\n how='left',\n left_on='service',\n right_on='service',\n suffixes=['', '_x'])\n\n return df[['service', 'http_latency_in', 'http_req_throughput_in', 'http_error_rate_in',\n 'inbound_conns', 'outbound_conns']]\n\n\ndef service_let_graph(start_time: str):\n ''' Compute a summary of traffic by requesting service, for requests on services\n in the current cluster.\n Args:\n @start_time: The timestamp of data to start at.\n '''\n df = pxviews.http_graph(start_time, px.now())\n df.window = px.DurationNanos(px.now() - px.parse_time(start_time))\n # Compute statistics about each edge of the service graph.\n df.request_throughput = df.num_requests / df.window\n df.inbound_throughput = df.req_bytes / df.window\n df.outbound_throughput = df.resp_bytes / df.window\n df.throughput_total = df.num_requests\n df.error_rate = px.Percent(df.num_errors / df.num_requests)\n\n return df[[\n 'responder_pod',\n 'requestor_pod',\n 'responder_service',\n 'requestor_service',\n 'responder_ip',\n 'requestor_ip',\n 'latency_p50',\n 'latency_p90',\n 'latency_p99',\n 'request_throughput',\n 'error_rate',\n 'inbound_throughput',\n 'outbound_throughput',\n 'throughput_total'\n ]]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"HTTP Service Map\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"service_let_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.RequestGraph\",\n \"requestorPodColumn\": \"requestor_pod\",\n \"responderPodColumn\": \"responder_pod\",\n \"requestorServiceColumn\": \"requestor_service\",\n \"responderServiceColumn\": \"responder_service\",\n \"requestorIPColumn\": \"requestor_ip\",\n \"responderIPColumn\": \"responder_ip\",\n \"p50Column\": \"latency_p50\",\n \"p90Column\": \"latency_p90\",\n \"p99Column\": \"latency_p99\",\n \"errorRateColumn\": \"error_rate\",\n \"requestsPerSecondColumn\": \"request_throughput\",\n \"inboundBytesPerSecondColumn\": \"inbound_throughput\",\n \"outboundBytesPerSecondColumn\": \"outbound_throughput\",\n \"totalRequestCountColumn\": \"throughput_total\"\n }\n },\n {\n \"name\": \"Nodes\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"nodes_for_cluster\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Namespaces\",\n \"position\": {\n \"x\": 6,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"namespaces_for_cluster\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Services\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"services_for_cluster\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Pods\",\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"pods_for_cluster\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Cluster Overview","LongDoc":"This view lists the namespaces and the node that are available on the current cluster.\n","orgID":"","hidden":false},"px/collect_agent_heaps":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef collect_pprofs(asid: int):\n df = px.GetAgentStatus()\n df.ip_address = px.pluck_array(px.split(df.ip_address, \":\"), 0)\n df.hostname_by_ip = px.pod_id_to_node_name(px.ip_to_pod_id(df.ip_address))\n df.hostname = px.select(df.hostname_by_ip == \"\", df.hostname, df.hostname_by_ip)\n df = df[['asid', 'hostname']]\n heap_stats = px._HeapGrowthStacks(asid)\n df = df.merge(heap_stats, how='inner', left_on='asid', right_on='asid')\n df.asid = df.asid_x\n return df[['asid', 'hostname', 'heap']]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"asid\",\n \"type\": \"PX_INT64\",\n \"description\": \"Whether to filter the results to a particular ASID\",\n \"defaultValue\": \"-1\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"collect_pprofs\",\n \"func\": {\n \"name\": \"collect_pprofs\",\n \"args\": [\n {\n \"name\": \"asid\",\n \"variable\": \"asid\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"collect_pprofs\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Collect Agent Heap Dumps","LongDoc":"Script useful for debugging kelvin and PEM memory footprint.","orgID":"","hidden":false},"px/cql_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' CQL Data Tracer\n\nThis script traces all CQL (Cassandra) data on the cluster.\n'''\nimport px\n\n\ndef cql_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='cql_events', start_time=start_time)\n\n # Add context.\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Translate opcode value to message type\n df.req_op = px.cql_opcode_name(df.req_op)\n df.resp_op = px.cql_opcode_name(df.resp_op)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n # Order columns.\n df = df['time_', 'source', 'destination', 'latency', 'req_op', 'req_body',\n 'resp_op', 'resp_body']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the CQL request.\n\n CQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the CQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the CQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"cql_data\",\n \"func\": {\n \"name\": \"cql_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"cql_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Sample CQL Data","LongDoc":"Shows a sample of CQL (Cassandra) requests in the cluster.","orgID":"","hidden":false},"px/cql_flow_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Cassandra Overview Map\n\nShows a graph of Cassandra requests in the cluster, with some latency information.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef cql_flow_graph(start_time: str, ns: px.Namespace, source_filter: str, destination_filter: str):\n\n df = px.DataFrame('cql_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter on namespace as specified by the user.\n df = df[df.namespace == ns]\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Create 10 ns bin for time_ column\n df.timestamp = px.bin(df.time_, window_ns)\n\n df = df.groupby(['timestamp', 'source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace']).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count),\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n\n df = df.groupby(['source', 'destination', 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n return df\n\n\ndef cql_summary_with_links(start_time: str, ns: px.Namespace, source_filter: str, destination_filter: str):\n\n df = cql_flow_graph(start_time, ns, source_filter, destination_filter)\n df = add_source_dest_links(df, start_time)\n df = df[['source', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput', 'throughput_total']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Cassandra request.\n\n Cassandra requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Cassandra request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Cassandra request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"cql_flow\",\n \"func\": {\n \"name\": \"cql_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"cql_summary_with_links\",\n \"func\": {\n \"name\": \"cql_summary_with_links\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Cassandra Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"cql_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"source\",\n \"toColumn\": \"destination\"\n },\n \"edgeWeightColumn\": \"request_throughput\",\n \"edgeColorColumn\": \"latency_p90\",\n \"edgeThresholds\": {\n \"mediumThreshold\": 100000000,\n \"highThreshold\": 500000000\n },\n \"edgeHoverInfo\": [\n \"latency_p50\",\n \"latency_p90\",\n \"latency_p99\",\n \"request_throughput\",\n \"throughput_total\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"cql_summary_with_links\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Cassandra Flow Graph","LongDoc":"Graph of Cassandra messages in the cluster, with latency stats.\n","orgID":"","hidden":false},"px/cql_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' CQL Pod LET metrics\n\nThis live view calculates the latency, error rate, and throughput\nof a pod's CQL requests.\n\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\n# K8s object is the abstraction to group on.\n# Options are ['pod', 'service'].\nk8s_object = 'pod'\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Column name used to split data into separate time series.\n# k8s_object column is renamed to this and is used in\n# visualization spec.\nsplit_series_name = 'k8s'\n# This is the name of the column corresponding to the CQL IP.\nip_col_name = 'CQL IP'\n# Temporary way to ensure px.Pod works as expected.\npx.Pod = str\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(50 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization speciciation, vis.json.\n# ----------------------------------------------------------------\ndef pod_cql_let(start_time: str, pod: px.Pod):\n \"\"\" Calculate LET time-series for CQL traffic per Pod, database pair.\n\n Calculates latency, error_rate, and throughput for each pod's\n connection to CQL databases.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor CQL LET.\n\n Returns: Returns the DataFrame contiaining LET time-series for CQL\n traffic for each database Pods talk to.\n \"\"\"\n df = cql_let_per_pod(start_time, pod)\n return df['time_', split_series_name, ip_col_name, 'latency_p50',\n 'latency_p90', 'latency_p99', 'error_rate', 'request_throughput']\n\n\ndef summary_cql_let(start_time: str, pod: px.Pod):\n \"\"\" Calculate LET summary for CQL traffic per Pod, database pair.\n\n Calculates latency, error_rate, and throughput for each pod's\n connection to CQL databases.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor CQL LET.\n\n Returns: Returns the DataFrame contiaining LET time-series for CQL\n traffic for each database Pods talk to.\n \"\"\"\n df = cql_let_per_pod(start_time, pod)\n summary_df = summarize_LET(df, [k8s_object, ip_col_name])\n return summary_df[[k8s_object, ip_col_name, \"request_throughput\",\n \"error_rate\", \"latency\", \"total_requests\"]]\n\n\ndef latency_histogram(start_time: str, pod: px.Pod):\n \"\"\" Computes a histogram of HTTP request latency.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc.\n\n Returns: DataFrame of the HTTP latency histogram for svcs that\n match @svc.\n \"\"\"\n # The data necessary to compute CQL LET information is located in the\n # cql_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='cql_events', start_time=start_time)\n df = format_cql_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to CQL conections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df[k8s_object], pod)]\n\n matching_df.request_latency = px.bin(matching_df.latency,\n latency_bin_size_ns)\n return matching_df.groupby('request_latency').agg(count=('time_', px.count))\n\n\ndef cmd_timeseries(start_time: str, pod: px.Pod):\n ''' Calculate CQL cmd timeseries for CQL traffic per connection to\n a CQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: The partial/full-name of the pod to monitor.\n\n Returns: Returns the DataFrame containing Redis cmd time-series for CQL\n traffic to a CQL database pod.\n '''\n df = px.DataFrame(table='cql_events', start_time=start_time)\n df = format_cql_table(df)\n df = df[px.contains(df[k8s_object], pod)]\n df.time_ = px.bin(df.time_, window_ns)\n df.req_op = px.cql_opcode_name(df.req_op)\n df = df.groupby(['time_', 'req_op']).agg(\n throughput_total=('req_op', px.count),\n )\n return df\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\n\n\ndef cql_let_per_pod(start_time: str, pod: px.Pod):\n \"\"\" Calculate LET time-series for CQL traffic per Pod, database pair.\n\n Calculates latency, error_rate, and throughput for each pod's\n connection to CQL databases.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor CQL LET.\n\n Returns: Returns the DataFrame contiaining LET time-series for CQL\n traffic for each database Pods talk to.\n \"\"\"\n # The data necessary to compute CQL LET information is located in the\n # cql_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='cql_events', start_time=start_time)\n df = format_cql_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to CQL conections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df[k8s_object], pod)]\n let_df = calc_cql_LET(\n matching_df, [k8s_object, 'timestamp', 'remote_addr'])\n let_df[split_series_name] = let_df[k8s_object]\n let_df[ip_col_name] = let_df.remote_addr\n return let_df\n\n\ndef format_events_table(df, latency_col):\n \"\"\" Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency_ms', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on \"cql_events\" and \"http_events\"\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n \"\"\"\n df.latency = df[latency_col]\n\n df.timestamp = px.bin(df.time_, window_ns)\n df[k8s_object] = df.ctx[k8s_object]\n df = df[df[k8s_object] != '']\n return df\n\n\ndef format_cql_table(df):\n \"\"\" Formats cql_events tables\n\n Runs events table universal formatting, creates a failure field\n marking which requests receive an error status code.\n\n Args:\n @df: the input cql_events table.\n\n Returns: formatted cql_events DataFrame.\n \"\"\"\n df = format_events_table(df, 'latency')\n df.failure = df['resp_op'] == 0\n\n return df\n\n\ndef format_LET_aggs(df):\n \"\"\" Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics that\n can be visualized. Latency quantile values need to be extracted from the\n quantiles struct, and then error_rate and request_throughput are calculated as\n a function of window size.\n\n\n This function represents logic shared by LET calculators for CQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total', 'error_rate_per_window', and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n \"\"\"\n\n df.latency_p50 = px.pluck_float64(df.latency_quantiles, 'p50')\n df.latency_p90 = px.pluck_float64(df.latency_quantiles, 'p90')\n df.latency_p99 = px.pluck_float64(df.latency_quantiles, 'p99')\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n df.error_rate = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n\n return df\n\n\ndef calc_cql_LET(df, groups):\n \"\"\" Calculates Latency, Error Rate, and Throughput on CQL events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups.\n\n Args:\n @df: the input cql_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n \"\"\"\n # All requests for errors and throughput\n et = df.groupby(groups).agg(\n throughput_total=('latency', px.count),\n error_rate_per_window=('failure', px.mean),\n )\n\n filt_df = df[df.resp_op != 0]\n # Calculate latency on all requests that return or return an err.\n lcy = filt_df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n )\n # Left join because et's groups are a strict superset of lcy.\n df = et.merge(lcy, how='left', left_on=groups,\n right_on=groups, suffixes=['', '_x'])\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef ip_to_svc_name(df, ip_col, svc_col_name):\n \"\"\" Map IP to svc name.\n\n Maps IP values stored in @ip_col into svc names to store into\n @svc_col_name.\n\n Args:\n @df: the input dataframe.\n @ip_col: the IP column to map from.\n @svc_col_name: the column name to assign the new svc values.\n\n Returns: DataFrame with the svc_col added.\n \"\"\"\n pod_id = 'pod_id'\n df[pod_id] = px.ip_to_pod_id(df[ip_col])\n df[svc_col_name] = px.pod_id_to_service_name(df[pod_id])\n return df.drop(pod_id)\n\n\ndef summarize_LET(let_df, groups):\n \"\"\" Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n \"\"\"\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n error_rate=('error_rate', px.mean),\n total_requests=('throughput_total', px.sum),\n latency=('latency_p50', px.mean),\n )\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to filter by for CQL request. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_cql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"p50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"p90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"RPS\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput per Message Type\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"cmd_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"throughput_total\",\n \"series\": \"req_op\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Message throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Error Rate\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Error Rate (%)\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Latency Histogram\",\n \"func\": {\n \"name\": \"latency_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.HistogramChart\",\n \"histogram\": {\n \"value\": \"request_latency\",\n \"prebinCount\": \"count\",\n \"maxbins\": 10,\n \"minstep\": 50.0\n },\n \"xAxis\": {\n \"label\": \"Request Latency\"\n },\n \"yAxis\": {\n \"label\": \"# of requests\"\n }\n },\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_cql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"CQL Pod LET metrics","LongDoc":"This live view calculates the latency, error rate, and throughput of a pod's CQL (Cassandra) requests.\n","orgID":"","hidden":false},"px/differential_flamegraph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\nnegate = False\n# TODO(ddelnano): negation requires switching the type of join from right to left\n# or returning a different DataFrame. This might not be possible with pxl's current\n# functionality, but this should be implemented once it's possible.\n\ndef merge_and_compute_delta(pod1, pod2, negate: bool):\n\n diff = pod1.merge(\n pod2,\n how='right',\n left_on='stack_trace',\n right_on='stack_trace'\n suffixes=['_1', '_2'],\n )\n # TODO(ddelnano): This needs to be switched with pod1 if the flamegraph should\n # be negated.\n percentage_agg = pod2.groupby(['pod']).agg(\n count=('count', px.sum),\n )\n diff.pod = px.select(negate, diff.pod_1, diff.pod_2)\n diff.stack_trace = px.select(negate, diff.stack_trace_1, diff.stack_trace_2)\n diff.stack_trace = px.replace(' ', diff.stack_trace, '')\n diff.count = px.select(negate, diff.count_1, diff.count_2)\n diff.delta = diff.count_2 - diff.count_1\n diff.delta = px.select(negate, px.negate(diff.delta), diff.delta)\n\n merged = diff.merge(\n percentage_agg,\n how='inner',\n left_on='pod',\n right_on='pod',\n suffixes=['', '_x']\n )\n merged.percent = 100 * merged.count / merged.count_x\n return merged\n\ndef differential_flamegraph(start_time: str, namespace: str, pod: str, baseline_pod: str):\n stack_traces = px.DataFrame(table='stack_traces.beta', start_time=start_time)\n stack_traces.namespace = stack_traces.ctx['namespace']\n stack_traces = stack_traces[stack_traces.namespace == namespace]\n stack_traces.node = px.Node(px._exec_hostname())\n stack_traces.pod = stack_traces.ctx['pod']\n stack_traces.keep_row = stack_traces.pod == baseline_pod\n stack_traces.keep_row = px.select(stack_traces.keep_row or stack_traces.pod == pod, True, False)\n stack_traces = stack_traces[stack_traces.keep_row]\n\n stack_traces = stack_traces.groupby(['node', 'namespace', 'pod', 'stack_trace_id']).agg(\n stack_trace=('stack_trace', px.any),\n count=('count', px.sum)\n )\n\n pod1 = stack_traces[stack_traces.pod == baseline_pod]\n pod1 = pod1.drop(['node', 'namespace', 'stack_trace_id'])\n\n pod2 = stack_traces[stack_traces.pod == pod]\n pod2 = pod2.drop(['node', 'namespace', 'stack_trace_id'])\n\n merged = merge_and_compute_delta(pod1, pod2, negate)\n return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The pod that will have its flamegraph analyzed compared to the baseline_pod\"\n },\n {\n \"name\": \"baseline_pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The pod to serve as the baseline. The resulting flamegraph will show the difference from this pod's profile.\"\n }\n ],\n \"globalFuncs\": [],\n \"widgets\": [\n {\n \"name\": \"Flamegraph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 6\n },\n \"func\": {\n \"name\": \"differential_flamegraph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n },\n {\n \"name\": \"baseline_pod\",\n \"variable\": \"baseline_pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.StackTraceFlameGraph\",\n \"stacktraceColumn\": \"stack_trace\",\n \"countColumn\": \"count\",\n \"percentageColumn\": \"percent\",\n \"podColumn\": \"pod\",\n \"differenceColumn\": \"delta\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Differential Flame Graph","LongDoc":"This live view shows a differential CPU flame graph. This is helpful in identifying what code paths have changed between deployments, different container instances, etc.\n","orgID":"","hidden":false},"px/dns_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' DNS Traffic Tracer\n\nThis script traces all DNS traffic on the cluster for a specified amount of time.\n'''\nimport px\n\n\ndef dns_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='dns_events', start_time=start_time)\n\n # Add context.\n df.node = df.ctx['node']\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n # Drop some columns.\n df = df['time_', 'source', 'destination', 'latency', 'req_header', 'req_body',\n 'resp_header', 'resp_body']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the DNS request.\n\n DNS requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the DNS request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the DNS request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"dns_data\",\n \"func\": {\n \"name\": \"dns_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"dns_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Raw DNS Data","LongDoc":"Show a sample of DNS traffic in the cluster.","orgID":"","hidden":false},"px/dns_flow_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' DNS Overview Map\nShows summary DNS requests in the cluster, with some latency information.\n'''\nimport px\n\n\n# kube-dns.kube-system.svc.cluster.local -\u003e kube-dns.kube-system\ndef format_nslookup_name(df):\n df.idx1 = px.find(df.to_entity, '.svc.cluster')\n leftovers = df[df.idx1 \u003c 0]\n df = df[df.idx1 \u003e= 0]\n df.to_entity = px.substring(df.to_entity, 0, df.idx1)\n return df.append(leftovers)\n\n\ndef dns_flow_graph(start_time: str, from_entity_filter: str, to_entity_filter: str):\n df = px.DataFrame('dns_events', start_time=start_time)\n\n # Client-side tracing only.\n df = df[df.trace_role == 1]\n df = df.drop(['trace_role'])\n\n # Add context.\n df.pod = df.ctx['pod']\n df.service = df.ctx['service']\n df.namespace = df.ctx['namespace']\n\n # Create table in drawer.\n px.debug(df, \"dns_events\")\n\n # Filter nodes for graph.\n df = df[not px.contains(df.pod, \"pl\")]\n df = df[not df.pod == \"\"]\n df = df[not df.remote_addr == \"-\"]\n\n # Specify from and to entities.\n df.from_entity = df.pod\n\n localhost_ip_regexp = r'127\\.0\\.0\\.[0-9]+'\n df.is_remote_addr_localhost = px.regex_match(localhost_ip_regexp, df.remote_addr)\n df.to_entity = px.select(df.is_remote_addr_localhost,\n px.upid_to_pod_name(df.upid),\n px.Service(px.nslookup(df.remote_addr)))\n\n df = df.drop(['upid', 'is_remote_addr_localhost'])\n\n # Reformat to_entity and from_entity for consistency between pods and services.\n df = format_nslookup_name(df)\n\n # Aggregate the connections.\n df = df.groupby(['from_entity', 'to_entity']).agg(\n latency_avg=('latency', px.mean),\n latency_max=('latency', px.max),\n count=('latency', px.count)\n )\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.from_entity, from_entity_filter)]\n df = df[px.contains(df.to_entity, to_entity_filter)]\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"from_entity_filter\",\n \"type\": \"PX_POD\",\n \"description\": \"The partial string to match the 'from_entity' (source pod).\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"to_entity_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'to_entity' (destination service).\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"dns_flow\",\n \"func\": {\n \"name\": \"dns_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"from_entity_filter\",\n \"variable\": \"from_entity_filter\"\n },\n {\n \"name\": \"to_entity_filter\",\n \"variable\": \"to_entity_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"DNS Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"dns_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"from_entity\",\n \"toColumn\": \"to_entity\"\n },\n \"edgeWeightColumn\": \"latency_avg\",\n \"edgeHoverInfo\": [\n \"latency_avg\",\n \"latency_max\",\n \"count\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 4,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"dns_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"DNS Flow Graph","LongDoc":"Overview of DNS requests in the cluster, with latency stats.\n","orgID":"","hidden":false},"px/dns_query_summary":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' DNS Overview Map\nShows DNS requests made by pods, clustered by name prefix.\n'''\nimport px\n\n\ndef dns_queries(start_time: str, namespace: str, pod_filter: str,\n query_filter: str, dns_server_filter: str):\n df = px.DataFrame('dns_events', start_time=start_time)\n\n # Client-side tracing only.\n df = df[df.trace_role == 1]\n df = df.drop(['trace_role'])\n\n # Add context.\n df.pod = df.ctx['pod']\n df.service = df.ctx['service']\n df.namespace = df.ctx['namespace']\n df = df.drop(['upid'])\n\n # Filter by namesapce\n df = df[px.contains(df.namespace, namespace)]\n\n # Extract some fields into their own columns for convenience.\n df.req_body = px.pluck(df.req_body, \"queries\")\n df.resp_body = px.pluck(df.resp_body, \"answers\")\n df.rcode = px.pluck_int64(df.resp_header, \"rcode\")\n df.resolved = px.contains(df.resp_body, \"name\")\n\n # Get the query prefix up to the first \".\"\n df.idxa = px.find(df.req_body, \"\\\"name\\\":\")\n df.idxz = px.find(df.req_body, \".\")\n df.qname = px.substring(df.req_body, df.idxa + 8, df.idxz - df.idxa - 8)\n df = df.drop(['idxa', 'idxz'])\n\n # Convert DNS IP to string.\n df.dns_server = px.nslookup(df.remote_addr)\n\n # Apply user-filters on the data.\n df = df[px.contains(df.pod, pod_filter)]\n df = df[px.contains(df.qname, query_filter)]\n df = df[px.contains(df.dns_server, dns_server_filter)]\n\n # Before aggregating, output individual requests to drawer.\n px.debug(df, \"events\")\n\n # Aggregate by each pod making a query to a particular DNS.\n df.nxdomain = df.rcode == 3\n df = df.groupby(['pod', 'dns_server', 'qname']).agg(\n num_requests=('resolved', px.count),\n num_resolved=('resolved', px.sum),\n num_nxdomain=('nxdomain', px.sum),\n )\n\n # Compute percentages.\n df.unresolved_rate = px.Percent((df.num_requests - df.num_resolved) / df.num_requests)\n df.nxdomain_rate = px.Percent((df.num_nxdomain) / df.num_requests)\n df.qgroup = df.qname + \" @\" + df.dns_server\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to query.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"pod_filter\",\n \"type\": \"PX_POD\",\n \"description\": \"Filter on DNS requests where the source pod contains the provided string.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"query_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"Filter on DNS requests where the query contains the provided string.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"dns_server_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"Filter on DNS requests where the target DNS server contains the provided string.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"dns_flow\",\n \"func\": {\n \"name\": \"dns_queries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"pod_filter\",\n \"variable\": \"pod_filter\"\n },\n {\n \"name\": \"query_filter\",\n \"variable\": \"query_filter\"\n },\n {\n \"name\": \"dns_server_filter\",\n \"variable\": \"dns_server_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"DNS Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"dns_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"pod\",\n \"toColumn\": \"qgroup\"\n },\n \"edgeWeightColumn\": \"total\",\n \"edgeColorColumn\": \"unresolved\",\n \"edgeHoverInfo\": [\n \"total\",\n \"unresolved\",\n \"nx_domain\"\n ],\n \"edgeThresholds\": {\n \"mediumThreshold\": 50,\n \"highThreshold\": 90\n },\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 4,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"dns_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"DNS Queries Overview","LongDoc":"Overview of DNS queries from pods in a namespace, grouped by the name being resolved and the rates of success.\n","orgID":"","hidden":false},"px/dx_evidence_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef dx_attack_graph(start_time: str, clickhouse_dsn: str, table: str):\n df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time)\n return df[['requestor_pod', 'responder_pod',\n 'requestor_service', 'responder_service',\n 'requestor_ip', 'responder_ip',\n 'weight', 'max_severity', 'confidence',\n 'edge_kind', 'condition', 'criteria', 'num_findings',\n 'investigation_id']]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"Start time of the window.\",\n \"defaultValue\": \"-15m\"\n },\n {\n \"name\": \"clickhouse_dsn\",\n \"type\": \"PX_STRING\",\n \"description\": \"ClickHouse DSN: user:pass@host:port/db.\",\n \"defaultValue\": \"forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db\"\n },\n {\n \"name\": \"table\",\n \"type\": \"PX_STRING\",\n \"description\": \"dx_attack_graph_malicious (default; rule-ins only — benign is NOT pulled from ClickHouse) or dx_attack_graph (full table, includes benign).\",\n \"defaultValue\": \"dx_attack_graph_malicious\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"dx_graph\",\n \"func\": {\n \"name\": \"dx_attack_graph\",\n \"args\": [\n {\"name\": \"start_time\", \"variable\": \"start_time\"},\n {\"name\": \"clickhouse_dsn\", \"variable\": \"clickhouse_dsn\"},\n {\"name\": \"table\", \"variable\": \"table\"}\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"DX Attack Graph\",\n \"position\": {\"x\": 0, \"y\": 0, \"w\": 12, \"h\": 5},\n \"globalFuncOutputName\": \"dx_graph\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"requestor_pod\",\n \"toColumn\": \"responder_pod\"\n },\n \"edgeWeightColumn\": \"weight\",\n \"edgeColorColumn\": \"max_severity\",\n \"edgeLabelColumn\": \"edge_kind\",\n \"edgeThresholds\": {\n \"mediumThreshold\": 3,\n \"highThreshold\": 4\n },\n \"edgeHoverInfo\": [\n \"edge_kind\",\n \"condition\",\n \"criteria\",\n \"weight\",\n \"max_severity\",\n \"confidence\",\n \"num_findings\",\n \"investigation_id\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Edges\",\n \"position\": {\"x\": 0, \"y\": 5, \"w\": 12, \"h\": 4},\n \"globalFuncOutputName\": \"dx_graph\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"DX Attack Graph","LongDoc":"Severity-weighted, all-protocol pod-to-pod attack graph for one dx-agent investigation. Renders attackgraph.Edge records emitted by dx with weight (sum of CRS evidence severity) on the edges and max_severity colouring the heat. v0 manual-load only — wires up to the dx_attack_graph ClickHouse / Pixie ingest in v1. See README.md in this directory.\n","orgID":"","hidden":false},"px/funcs":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef map_funcs():\n # Gets a list of the regular functions.\n return px.GetUDFList()\n\n\ndef agg_funcs():\n # Gets a list of the aggregate functions.\n return px.GetUDAList()\n\n\ndef table_funcs():\n # Gets a list of the table generating functions.\n return px.GetUDTFList()\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [],\n \"widgets\": [\n {\n \"name\": \"Agg Functions\",\n \"func\": {\n \"name\": \"agg_funcs\",\n \"args\": []\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n }\n },\n {\n \"name\": \"Table Functions\",\n \"func\": {\n \"name\": \"table_funcs\",\n \"args\": []\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 3\n }\n },\n {\n \"name\": \"Map Functions\",\n \"func\": {\n \"name\": \"map_funcs\",\n \"args\": []\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 4\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Get a list all of all funcs available in Pixie.","LongDoc":"Gets a list all of the funcs available in Pixie.\n","orgID":"","hidden":false},"px/http_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' HTTP Data Tracer\n\nThis script traces all HTTP/HTTP2 data on the cluster.\n'''\nimport px\n\n\ndef http_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='http_events', start_time=start_time)\n\n # Add context.\n df.node = df.ctx['node']\n df.pid = px.upid_to_pid(df.upid)\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n # Order columns.\n df = df['time_', 'source', 'destination', 'latency', 'major_version', 'req_path',\n 'req_method', 'req_headers', 'req_body', 'req_body_size', 'resp_status',\n 'resp_message', 'resp_headers', 'resp_body', 'resp_body_size']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the HTTP request.\n\n HTTP requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the HTTP request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the HTTP request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"http_data\",\n \"func\": {\n \"name\": \"http_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"http_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"HTTP Data","LongDoc":"Shows most recent HTTP messages in the cluster.","orgID":"","hidden":false},"px/http_data_filtered":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' HTTP Data Tracer\n\nThis script traces all HTTP/HTTP2 data on the cluster for a specified amount of time.\nAn optional filter prints only those traces that include the specified service name.\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Script variables\n# ----------------------------------------------------------------\n\nmax_num_records = 100\n\n# ----------------------------------------------------------------\n# Implementation\n# ----------------------------------------------------------------\n\n\ndef http_data(start_time: str, svc: px.Service, pod: px.Pod, req_path: str, status_code: int):\n df = px.DataFrame(table='http_events', select=['time_', 'upid', 'remote_addr', 'remote_port',\n 'req_method', 'req_path', 'resp_status',\n 'resp_body', 'latency'],\n start_time=start_time)\n\n df.svc = df.ctx['service']\n df.pod = df.ctx['pod']\n df = df.drop(columns=['upid'])\n df = df[df.resp_status == status_code]\n df = df[px.contains(df.svc, svc) and (px.contains(df.pod, pod)\n and px.contains(df.req_path, req_path))]\n df = df.head(n=max_num_records)\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"svc\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The full/partial name of the service to get stats for. Format: ns/svc_name\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_STRING\",\n \"description\": \"The full/partial name of the pod to get stats for. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"req_path\",\n \"type\": \"PX_STRING\",\n \"description\": \"The full/partial name of the request_path.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"status_code\",\n \"type\": \"PX_INT64\",\n \"description\": \"The HTTP response status code.\",\n \"defaultValue\": \"200\"\n },\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-30s\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"HTTP Data\",\n \"func\": {\n \"name\": \"http_data\",\n \"args\": [\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n },\n {\n \"name\": \"req_path\",\n \"variable\": \"req_path\"\n },\n {\n \"name\": \"status_code\",\n \"variable\": \"status_code\"\n },\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n }\n }\n ],\n \"widgets\": []\n}\n","placement":"","ShortDoc":"Sample HTTP Data","LongDoc":"Show a sample of HTTP requests in the Cluster filtered by service, pod, request path \u0026 response status code.","orgID":"","hidden":false},"px/http_post_requests":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' HTTP Data Tracer\n\nThis script traces all HTTP/HTTP2 data on the cluster for a specified amount of time.\nAn optional filter prints only those traces that include the specified service name.\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Script variables\n# ----------------------------------------------------------------\n\nservice_matcher = ''\nstart_time = '-30s'\nmax_num_records = 100\n\n# ----------------------------------------------------------------\n# Implementation\n# ----------------------------------------------------------------\n\ndf = px.DataFrame(table='http_events', select=['time_', 'upid', 'remote_addr', 'remote_port',\n 'req_method', 'req_path', 'resp_status',\n 'resp_body', 'latency'],\n start_time=start_time)\n\ndf = df[df.req_method == 'POST']\ndf.service = df.ctx['service']\ndf = df.drop(columns=['upid'])\n\ndf = df[px.contains(df['service'], service_matcher)]\ndf = df.head(n=max_num_records)\n\npx.display(df)\n","vis":"","placement":"","ShortDoc":"Sample HTTP POST Data","LongDoc":"Show a sample of HTTP requests in the Cluster that have method POST.","orgID":"","hidden":false},"px/http_request_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\nt1 = px.DataFrame(table='http_events', start_time='-30s')\n\nt1.service = t1.ctx['service']\nt1.failure = t1.resp_status \u003e= 400\nwindow = px.DurationNanos(px.seconds(5))\nt1.range_group = px.bin(t1.time_, window)\n\nquantiles_agg = t1.groupby('service').agg(\n latency_quantiles=('latency', px.quantiles),\n errors=('failure', px.mean),\n throughput_total=('resp_status', px.count),\n)\nquantiles_agg.errors = px.Percent(quantiles_agg.errors)\n\nquantiles_agg.latency_p50 = px.DurationNanos(px.floor(\n px.pluck_float64(quantiles_agg.latency_quantiles, 'p50')))\nquantiles_agg.latency_p90 = px.DurationNanos(px.floor(\n px.pluck_float64(quantiles_agg.latency_quantiles, 'p90')))\nquantiles_agg.latency_p99 = px.DurationNanos(px.floor(\n px.pluck_float64(quantiles_agg.latency_quantiles, 'p99')))\nquantiles_table = quantiles_agg[['service', 'latency_p50',\n 'latency_p90', 'latency_p99', 'errors', 'throughput_total']]\n\n# The Range aggregate to calcualte the requests per second.\nrange_agg = t1.groupby(['service', 'range_group']).agg(\n requests_per_window=('resp_status', px.count),\n)\n\nrps_table = range_agg.groupby('service').agg(\n request_throughput=('requests_per_window', px.mean))\n\njoined_table = quantiles_table.merge(rps_table,\n how='inner',\n left_on=['service'],\n right_on=['service'],\n suffixes=['', '_x'])\n\njoined_table['latency(p50)'] = joined_table.latency_p50\njoined_table['latency(p90)'] = joined_table.latency_p90\njoined_table['latency(p99)'] = joined_table.latency_p99\njoined_table['throughput'] = joined_table.request_throughput / window\njoined_table['throughput total'] = joined_table.throughput_total\n\njoined_table = joined_table[[\n 'service',\n 'latency(p50)',\n 'latency(p90)',\n 'latency(p99)',\n 'errors',\n 'throughput',\n 'throughput total']]\njoined_table = joined_table[joined_table.service != '']\npx.display(joined_table)\n","vis":"","placement":"","ShortDoc":"HTTP Requests Statistics by Service","LongDoc":"HTTP request statistics aggregated by Service","orgID":"","hidden":false},"px/http_trace_id":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' HTTP Trace Id Tracer\n\nThis script traces all HTTP/HTTP2 data on the cluster that includes an Otel traceparent http header.\nIn addition, it allows filtering by trace id so all of the spans contained within that trace can\nbe viewed.\n'''\nimport px\n\n\ndef http_trace_data(start_time: str, trace_header_name: str, trace_id_filter: str):\n\n df = px.DataFrame(table='http_events', start_time=start_time)\n\n # Add context.\n df.node = df.ctx['node']\n df.pid = px.upid_to_pid(df.upid)\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Determine the trace_id from the {trace_header_name} http header. Otel's trace\n # header (traceparent) includes more than the trace id and requires further\n # parsing, while others (X-B3-Trace-Id) the value can be used wholesale.\n df.trace_id = px.pluck(df.req_headers, trace_header_name)\n df.trace_id = px.select(\n trace_header_name != 'traceparent', df.trace_id, px.replace(\n r\"[A-Za-z0-9]+\\-([A-Za-z0-9]+)\\-[A-Za-z0-9]+\\-[A-Za-z0-9]+\",\n df.trace_id, \"\\\\1\"))\n\n # Filter out requests that don't include a trace id.\n df = df[df.trace_id != \"\"]\n\n df = add_trace_id_link(df, start_time)\n\n df = df[px.contains(df.trace_id, trace_id_filter)]\n\n # Order columns.\n df = df['time_', 'trace_id', 'source', 'destination', 'latency', 'major_version', 'req_path',\n 'req_method', 'req_headers', 'req_body', 'req_body_size', 'resp_status',\n 'resp_message', 'resp_headers', 'resp_body', 'resp_body_size']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the HTTP request.\n\n HTTP requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the HTTP request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the HTTP request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_trace_id_link(df, start_time: str):\n '''Modifies the trace_id column to provide a deeplink in the UI to a filtered\n view of just the requests related to that trace id. This function assumes that\n any data frames that have empty trace ids have been filtered prior.\n '''\n\n # Use 2x the original start_time to avoid allowing a user to load the script for\n # a specified trace id when the requests involved have aged out of their original\n # start_time window. If PxL supported specifying the sort order of the table visualization\n # (descending time), this wouldn't be a concern.\n extended_start_time = px.format_duration(2 * px.parse_duration(start_time))\n\n df.trace_id_link = px.script_reference(df.trace_id, 'px/http_trace_id', {\n 'start_time': extended_start_time,\n 'trace_id_filter': df.trace_id,\n })\n df.trace_id = df.trace_id_link\n\n df = df.drop(['trace_id_link'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-30m\"\n },\n {\n \"name\": \"trace_header_name\",\n \"type\": \"PX_STRING\",\n \"description\": \"The HTTP header that contains the distributed trace id. Defaults to Otel's traceparent header\",\n \"defaultValue\": \"traceparent\"\n },\n {\n \"name\": \"trace_id_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"Trace id.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"http_trace_data\",\n \"func\": {\n \"name\": \"http_trace_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"trace_header_name\",\n \"variable\": \"trace_header_name\"\n },\n {\n \"name\": \"trace_id_filter\",\n \"variable\": \"trace_id_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"http_trace_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Sample HTTP Trace Data","LongDoc":"Shows HTTP requests with trace IDs in the cluster. Allows configuring which header contains the trace ID and filtering requests by trace ID.","orgID":"","hidden":false},"px/inbound_conns":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Inbound Connections\n\nThis script lists connections made from IPs that are not within the k8s cluster.\n'''\nimport px\n\n\ndef inbound_conns(start_time: str, ip_filter: str):\n df = px.DataFrame(table='conn_stats', start_time=start_time)\n\n df.namespace = df.ctx['namespace']\n df.pod = df.ctx['pod']\n\n # Filter for inbound traffic only.\n # Trace-role of 2 means server-side tracing.\n df = df[df.trace_role == 2]\n\n # Filter out any connections from known pods.\n df.remote_pod_id = px.ip_to_pod_id(df.remote_addr)\n df.remote_service_id = px.ip_to_service_id(df.remote_addr)\n df = df[df.remote_pod_id == '' and df.remote_service_id == '']\n\n # Filter out connections from localhost.\n df = df[not df.remote_addr == '127.0.0.1']\n\n # Apply user supplied filter.\n df = df[px.contains(df.remote_addr, ip_filter)]\n\n # Calculate connection stats for each process for each unique pod / remote_addr pair.\n df = df.groupby(['pod', 'upid', 'remote_addr']).agg(\n # The fields below are counters per UPID, so we take\n # the min (starting value) and the max (ending value) to subtract them.\n conn_open_min=('conn_open', px.min),\n conn_open_max=('conn_open', px.max),\n bytes_sent_min=('bytes_sent', px.min),\n bytes_sent_max=('bytes_sent', px.max),\n bytes_recv_min=('bytes_recv', px.min),\n bytes_recv_max=('bytes_recv', px.max),\n last_activity_time=('time_', px.max)\n )\n\n # Calculate connection stats over the time window.\n df.conn_open = df.conn_open_max - df.conn_open_min\n df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n\n # Calculate connection stats for each unique pod / remote_addr pair. Since there\n # may be multiple processes per pod we perform an additional aggregation to\n # consolidate those into one entry.\n df = df.groupby(['pod', 'remote_addr']).agg(\n conn_open=('conn_open', px.sum),\n bytes_sent=('bytes_sent', px.sum),\n bytes_recv=('bytes_recv', px.sum),\n last_activity_time=('last_activity_time', px.max)\n )\n\n return df[['pod', 'remote_addr', 'conn_open', 'bytes_sent', 'bytes_recv', 'last_activity_time']]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-24h\"\n },\n {\n \"name\": \"ip_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"IP filter, using a substring match.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"inbound_conns\",\n \"func\": {\n \"name\": \"inbound_conns\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ip_filter\",\n \"variable\": \"ip_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"inbound_conns\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Inbound Connections","LongDoc":"Shows a list of connections originating from endpoints outside the k8s cluster.","orgID":"","hidden":false},"px/ip":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef ip_info(start_time: str, ip: str):\n '''\n Computes the total number of bytes sent/received to this IP.\n Also returns metadata about this IP address.\n '''\n df = pod_traffic_to_ip(start_time, ip)\n df = df.agg(\n bytes_per_s_from_ip=('bytes_per_s_from_ip', px.sum),\n bytes_per_s_to_ip=('bytes_per_s_to_ip', px.sum)\n )\n df.resolved_domain = px.nslookup(ip)\n df.ip_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(ip))\n df.ip_service = px.service_id_to_service_name(px.ip_to_service_id(ip))\n df.ip_node = px.pod_id_to_node_name(px.ip_to_pod_id(ip))\n return df\n\n\ndef pod_traffic_to_ip(start_time: str, ip: str):\n '''\n Computes the total number of bytes sent/received to this IP by pod.\n Does not take trace_role into account, just computes the bytes sent by the pod\n and received by the pod to the IP address.\n '''\n df = traffic_involving_ip(start_time, ip)\n return df.groupby(['pod']).agg(\n bytes_per_s_from_ip=('bytes_per_s_from_ip', px.sum),\n bytes_per_s_to_ip=('bytes_per_s_to_ip', px.sum),\n total_bytes_per_s=('total_bytes_per_s', px.sum)\n )\n\n\ndef net_flow_graph(start_time: str, ip: str):\n '''\n Returns the graph representation of traffic from the cluster to the input IP address.\n Takes trace_role into account so that we can set the requestor and responder.\n '''\n df = traffic_involving_ip(start_time, ip)\n df.requestor = px.select(df.trace_role == 2, ip, df.pod)\n df.responder = px.select(df.trace_role == 2, df.pod, ip)\n df.requestor_sent_bytes_per_s = px.select(df.trace_role == 2,\n df.bytes_per_s_from_ip,\n df.bytes_per_s_to_ip)\n df.responder_sent_bytes_per_s = px.select(df.trace_role == 2,\n df.bytes_per_s_to_ip,\n df.bytes_per_s_from_ip)\n return df[['requestor', 'responder', 'requestor_sent_bytes_per_s',\n 'responder_sent_bytes_per_s', 'total_bytes_per_s']]\n\n\ndef traffic_involving_ip(start_time: str, ip: str):\n '''\n This is a helper function to return the traffic involving the input IP address.\n Specifically, it returns traffic from within the cluster that talks to that IP.\n It returns the bytes sent/received/total by pod that talks to the input IP.\n This function does not consider trace_role, that should be considered by any\n downstream consumer of this function.\n '''\n df = px.DataFrame('conn_stats', start_time=start_time)\n df = df[df.remote_addr == ip]\n\n # Store the pod/node/service talking to this IP.\n df.pod = df.ctx['pod']\n df.node = df.ctx['node']\n df.service = df.ctx['service']\n\n # Use aggregate to pick the first and last sample for any given client-server pair.\n # We do this by picking the min/max of the stats, since they are all counters.\n agg_df = df.groupby(['pod', 'node', 'service', 'upid', 'trace_role']).agg(\n bytes_sent_min=('bytes_sent', px.min),\n bytes_sent_max=('bytes_sent', px.max),\n bytes_recv_min=('bytes_recv', px.min),\n bytes_recv_max=('bytes_recv', px.max),\n )\n agg_df.bytes_sent_by_pod = agg_df.bytes_sent_max - agg_df.bytes_sent_min\n agg_df.bytes_recv_by_pod = agg_df.bytes_recv_max - agg_df.bytes_recv_min\n agg_df.total_bytes = agg_df.bytes_sent_by_pod + agg_df.bytes_recv_by_pod\n agg_df = agg_df.drop(['bytes_sent_max', 'bytes_sent_min', 'bytes_recv_max', 'bytes_recv_min'])\n\n # Since there may be multiple processes per pod (one per upid),\n # perform an additional aggregation to consolidate those into one entry.\n agg_df = agg_df.groupby(['pod', 'node', 'service', 'trace_role']).agg(\n bytes_sent_by_pod=('bytes_sent_by_pod', px.sum),\n bytes_recv_by_pod=('bytes_recv_by_pod', px.sum),\n total_bytes=('total_bytes', px.sum),\n )\n # Find the time window. Intentionally uses the pre-aggregated table, df.\n time_window = df.agg(\n time_min=('time_', px.min),\n time_max=('time_', px.max),\n )\n time_window.time_delta = px.DurationNanos(time_window.time_max - time_window.time_min)\n time_window = time_window.drop(['time_min', 'time_max'])\n\n res = agg_df.merge(time_window, how='inner', left_on=[], right_on=[])\n\n # Compute as rates. Notice the intentional inversion to change pod-\u003eip.\n res.bytes_per_s_from_ip = res.bytes_recv_by_pod / res.time_delta\n res.bytes_per_s_to_ip = res.bytes_sent_by_pod / res.time_delta\n res.total_bytes_per_s = res.total_bytes / res.time_delta\n return res.drop(['time_delta'])\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"ip\",\n \"type\": \"PX_STRING\",\n \"description\": \"The IP address to see traffic to/from in the cluster.\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Net Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"requestor\",\n \"toColumn\": \"responder\"\n },\n \"edgeWeightColumn\": \"total_bytes_per_s\",\n \"edgeColorColumn\": \"total_bytes_per_s\",\n \"edgeHoverInfo\": [\n \"total_bytes_per_s\",\n \"requestor_sent_bytes_per_s\",\n \"responder_sent_bytes_per_s\"\n ],\n \"edgeLength\": 500\n },\n \"func\": {\n \"name\": \"net_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ip\",\n \"variable\": \"ip\"\n }\n ]\n }\n },\n {\n \"name\": \"IP Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 4,\n \"w\": 12,\n \"h\": 2\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"func\": {\n \"name\": \"ip_info\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ip\",\n \"variable\": \"ip\"\n }\n ]\n }\n },\n {\n \"name\": \"Pod Traffic to IP\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 4\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"func\": {\n \"name\": \"pod_traffic_to_ip\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ip\",\n \"variable\": \"ip\"\n }\n ]\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Traffic to/from IP","LongDoc":"This view displays a summary of the traffic from the cluster to the input IP address.\n","orgID":"","hidden":false},"px/jvm_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\nbytes_per_mb = 1024 * 1024\n\ndf = px.DataFrame(table='jvm_stats', start_time='-1m')\n\n# Add additional fields\ndf.pid = px.upid_to_pid(df.upid)\ndf.cmdline = px.upid_to_cmdline(df.upid)\ndf.used_heap_size = px.Bytes(df.used_heap_size)\ndf.total_heap_size = px.Bytes(df.total_heap_size)\ndf.max_heap_size = px.Bytes(df.max_heap_size)\ndf = df[['time_', 'pid', 'used_heap_size', 'total_heap_size', 'max_heap_size', 'cmdline']]\npx.display(df)\n","vis":"","placement":"","ShortDoc":"JVM stats","LongDoc":"JVM stats for Java processes running on the cluster","orgID":"","hidden":false},"px/jvm_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' jvm per Pod and filtered by Node.\n\nThis live view summarizes the CPU percentage and network bytes\nreceived and transmitted per Pod. On top of that, you can\nisolate data to only a single node.\n\nNotes:\n* Setting node_name is not exclusive at the moment.\n* Setting pod is not exclusive at the moment.\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\n# K8s object is the abstraction to group on.\n# Options are ['pod', 'service'].\nk8s_object = 'pod'\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Name of the node column to display.\nnode_col = 'node_name'\n\nsplit_series_name = 'k8s'\npx.Node = str\npx.Pod = str\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization speciciation, vis.json.\n# ----------------------------------------------------------------\ndef node_table(start_time: str, node_name: px.Node, pod: px.Pod):\n \"\"\" Gets the nodes that are picked up by the node and pod filters.\n\n Args:\n @start_time Starting time of the data to examine.\n @node_name: The partial name of the node(s) to display data for.\n @pod: The partial name of the pod(s) to display data for.\n\n Returns: A DataFrame that displays all nodes matching the node\n and pod filters.\n \"\"\"\n df = filtered_process_stats(start_time, node_name, pod)\n # Aggregate on nodes to get unique node values.\n nodes = df.groupby(node_col).agg(cc=(node_col, px.count))\n return nodes.drop('cc')\n\n\ndef jvm_stats(start_time: str, node_name: px.Node, pod: px.Pod):\n \"\"\" Gets the JVM stats of pods that run on the matched nodes\n and that match the pod name.\n\n Args:\n @start_time Starting time of the data to examine.\n @node_name: The partial name of the node(s) to display data for.\n @pod: The partial name of the pod(s) to display data for.\n\n Returns: A DataFrame of jvm stats per pod matched in the passed\n in filters.\n \"\"\"\n df = filtered_process_stats(start_time, node_name, pod)\n # Calculate the CPU usage per window.\n jvm_df = calculate_jvm(df, window_ns)\n jvm_df[split_series_name] = jvm_df[k8s_object]\n return jvm_df\n\n\ndef cmd_pod_table(start_time: str, node_name: px.Node, pod: px.Pod):\n \"\"\" Gets the pods that are picked up by the node and pod filters.\n\n Args:\n @start_time Starting time of the data to examine.\n @node_name: The partial name of the node(s) to display data for.\n @pod: The partial name of the pod(s) to display data for.\n\n Returns: A DataFrame that displays all pods matching the node\n and pod filters.\n \"\"\"\n df = filtered_process_stats(start_time, node_name, pod)\n df.cmdline = px.upid_to_cmdline(df.upid)\n # Aggregate on nodes to get unique node values.\n pods = df.groupby([k8s_object, 'cmdline']).agg(cc=(k8s_object, px.count))\n return pods.drop('cc')\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef filtered_process_stats(start_time: str, node_name: px.Node, pod: px.Pod):\n \"\"\" Helper function that filters the process_stats tabel for data that\n matches the pod and node filters.\n\n Args:\n @start_time Starting time of the data to examine.\n @node_name: The partial name of the node(s) to display data for.\n @pod: The partial name of the pod(s) to display data for.\n\n Returns: A process_stats data formatted for other functions and\n filtered by matching pods located on nodes that match @node_name.\n\n \"\"\"\n\n # Load and prepare the process_stats table to calculate CPU usage\n # by pod.\n process_df = px.DataFrame(table='jvm_stats', start_time=start_time)\n process_df = format_jvm_table(process_df, window_ns)\n process_df = filter_stats(process_df, node_name, pod)\n return process_df\n\n\ndef format_stats_table(df, window_size):\n \"\"\" Format data and add semantic columns in stats tables.\n\n Adds a binned timestamp field to aggregate on.\n Works on \"process_stats\" and \"net_stats\"\n\n Args:\n @df: the input stats table.\n @window_size: the size of the window in seconds to aggregate on.\n\n Returns: formatted stats DataFrame.\n \"\"\"\n df.timestamp = px.bin(df.time_, window_size)\n return df\n\n\ndef format_jvm_table(df, window_size):\n \"\"\" Formats jvm_stats table.\n\n Maps in a pod column, node column and converts\n process-related metrics (cpu and memory) into a more readable\n format.\n\n Args:\n @df: the input process_stats table.\n @window_size: the size of the window in seconds to aggregate on.\n\n Returns: formatted process_stats DataFrame.\n \"\"\"\n df = format_stats_table(df, window_size)\n df[node_col] = df.ctx['node_name']\n df[k8s_object] = df.ctx[k8s_object]\n return df\n\n\ndef format_network_table(df, window_size):\n \"\"\" Formats network_stats table.\n\n Maps in a pod column, node column and converts\n network-related metrics (transmitted and received bytes) into\n a more readable format.\n\n Args:\n @df: the input network_stats table.\n @window_size: the size of the window in seconds to\n aggregate on.\n\n Returns: formatted network_stats DataFrame.\n \"\"\"\n df = format_stats_table(df, window_size)\n df[node_col] = px.pod_id_to_node_name(df.pod_id)\n df[k8s_object] = px.pod_id_to_pod_name(df.pod_id)\n return df\n\n\ndef filter_stats(df, node_filter, k8s_filter):\n \"\"\" Filter stats tables based that match the semantic values.\n\n Keep data from nodes that match @node_filter and\n k8s_objects that match @k8s_filter.\n\n Args:\n @df: the input process table.\n @node_filter: the partial name of nodes to match.\n @k8s_filter: the partial name of k8s_object to match.\n\n Returns: filtered stats DataFrame.\n \"\"\"\n df = df[px.contains(df[node_col], node_filter)]\n df = df[px.contains(df[k8s_object], k8s_filter)]\n return df\n\n\ndef calculate_jvm(df, window_size):\n \"\"\" Calculate jvm statistics per window per k8s_object.\n\n Calculates the value of critical JVM stats over each window\n of size window_size. The GC time metrics (young_gc_time and\n full_gc_time metrics) are monotically increasing counters,\n so we need to get the beginning and end values in the window.\n The other metrics are not counters, so we just take the mean\n over the window.\n\n Because jvm_stats is sharded by UPID (Unique PID), you must\n calculate this value by PID first, then sum up per k8s_object\n to get accurate results.\n\n Args:\n @df: the input jvm_stats table.\n @window_size: the size of the window in seconds to\n aggregate on.\n\n Returns: Calculated JVM stats per window and pod/svc.\n \"\"\"\n df.used_heap_size = px.Bytes(df.used_heap_size)\n df.total_heap_size = px.Bytes(df.total_heap_size)\n df.max_heap_size = px.Bytes(df.max_heap_size)\n # Aggregate over each UPID, k8s_object, and window.\n by_upid = df.groupby(['upid', k8s_object, 'timestamp']).agg(\n young_gc_time_max=('young_gc_time', px.max),\n young_gc_time_min=('young_gc_time', px.min),\n full_gc_time_max=('full_gc_time', px.max),\n full_gc_time_min=('full_gc_time', px.min),\n used_heap_size=('used_heap_size', px.mean),\n total_heap_size=('total_heap_size', px.mean),\n max_heap_size=('max_heap_size', px.mean),\n )\n\n # Conver the counter metrics into accumulated values over the window.\n by_upid.young_gc_time = by_upid.young_gc_time_max - by_upid.young_gc_time_min\n by_upid.full_gc_time = by_upid.full_gc_time_max - by_upid.full_gc_time_min\n\n # Then aggregate process individual process metrics into the overall\n # k8s_object.\n per_k8s = by_upid.groupby([k8s_object, 'timestamp']).agg(\n young_gc_time=('young_gc_time', px.sum),\n full_gc_time=('full_gc_time', px.sum),\n used_heap_size=('used_heap_size', px.sum),\n max_heap_size=('max_heap_size', px.sum),\n total_heap_size=('total_heap_size', px.sum),\n )\n per_k8s.young_gc_time = px.DurationNanos(per_k8s.young_gc_time)\n per_k8s.full_gc_time = px.DurationNanos(per_k8s.full_gc_time)\n\n # Finally, calculate total (kernel + user time) percentage used over window.\n per_k8s['time_'] = per_k8s['timestamp']\n return per_k8s\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"node_name\",\n \"type\": \"PX_STRING\",\n \"description\": \"The full/partial hostname to filter by\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to use as filter. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"JVM\",\n \"func\": {\n \"name\": \"jvm_stats\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node_name\",\n \"variable\": \"node_name\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Young GC Time\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"JVM\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"young_gc_time\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Young GC Time\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Full GC Time\",\n \"position\": {\n \"x\": 6,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"JVM\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"full_gc_time\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Full GC Time\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Used Heap Size\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"JVM\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"used_heap_size\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Used Heap Size\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Max Heap Size\",\n \"position\": {\n \"x\": 6,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"JVM\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"max_heap_size\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Max Heap Size (mb)\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Total Heap Size\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"JVM\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_heap_size\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Total Heap Size (mb)\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Processes that use JVM\",\n \"position\": {\n \"x\": 6,\n \"y\": 6,\n \"w\": 6,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"cmd_pod_table\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node_name\",\n \"variable\": \"node_name\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"JVM Stats per Pod","LongDoc":"Returns the JVM Stats per Pod. You can filter this by node.\n","orgID":"","hidden":false},"px/kafka_consumer_rebalancing":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Kafka Consumer Rebalancing Events\n\nVisualizes the most recent Kafka consumer rebalancing events, with delay.\n'''\n\nimport px\n\n\ndef kafka_join_sync_group_data(start_time: str):\n '''\n Get the raw JoinGroup and SyncGroup events from the kafka table.\n '''\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df = add_source_dest_columns(df)\n\n df.req_cmd = px.kafka_api_key_name(df.req_cmd)\n df = df['time_', 'source', 'destination', 'remote_port', 'req_cmd',\n 'req_body', 'resp', 'latency']\n\n # Filter JoinGroup and SyncGroup events.\n df = df[df.req_cmd == 'JoinGroup' or df.req_cmd == 'SyncGroup']\n\n return df\n\n\ndef kafka_join_sync_group_events(start_time: str, consumer_group_id: str):\n '''\n Filter the raw JoinGroup and SyncGroup events with a given group_id.\n '''\n df = kafka_join_sync_group_data(start_time)\n\n df = get_and_filter_group_id(df, consumer_group_id)\n\n df = df.drop('group_id')\n return df\n\n\ndef kafka_group_ids(start_time: str):\n '''\n Computes a list of consumer groups and the number of active members in each group.\n '''\n df = kafka_join_sync_delay(start_time, '')\n df.generation_id = px.atoi(df.generation_id, -1)\n df = df[df.generation_id != -1]\n\n # Calculate the number of members in each group.\n df = df.groupby(['group_id', 'generation_id']).agg(num_members=('generation_id', px.count))\n\n # Calculate the max generation_id in each group.\n df_max_generation_id = df.groupby('group_id').agg(generation_id=('generation_id', px.max))\n\n # Join the two tables to get the most recent number of members in each group.\n df = df.merge(df_max_generation_id,\n how='inner',\n left_on=['group_id', 'generation_id'],\n right_on=['group_id', 'generation_id'],\n suffixes=['', '_x'])\n df = df['group_id', 'num_members']\n return df\n\n\ndef kafka_join_sync_delay(start_time: str, consumer_group_id: str):\n '''\n Computes the delay from the JoinGroup Request to SyncGroup Response. During this delay,\n consumers in the group are stopped due to group rebalancing.\n '''\n df = kafka_join_sync_group_data(start_time)\n df = get_and_filter_group_id(df, consumer_group_id)\n\n df_join = df[df.req_cmd == 'JoinGroup']\n df_sync = df[df.req_cmd == 'SyncGroup']\n\n # The generation_id is incremented after each successful rebalancing.\n df_join.generation_id = px.pluck(df_join.resp, 'generation_id')\n df_sync.generation_id = px.pluck(df_sync.req_body, 'generation_id')\n\n # The member_id is the unique id for each consumer in a consumer group.\n df_join.member_id = px.pluck(df_join.resp, 'member_id')\n df_sync.member_id = px.pluck(df_sync.req_body, 'member_id')\n\n # Filter out invalid generation_ids and member_ids.\n df_join = df_join[df_join.generation_id != '-1' and df_join.member_id != \"\"]\n df_sync = df_sync[df_sync.generation_id != '-1' and df_sync.member_id != \"\"]\n\n # Merge on group_id, generation_id, and member_id.\n df = df_join.merge(df_sync,\n how='inner',\n left_on=['group_id', 'generation_id', 'member_id'],\n right_on=['group_id', 'generation_id', 'member_id'],\n suffixes=['', '_sync'])\n\n # Delay is the difference between join and sync request plus sync request's latency.\n df.delay = df.time__sync - df.time_ + df.latency_sync\n # Under normal circumstances, delay should never be smaller than zero. This is a safety check\n # to make sure the data shown makes sense.\n df.delay = px.DurationNanos(px.select(df.delay \u003c 0, 0, df.delay))\n df = df['time_', 'group_id', 'generation_id', 'member_id', 'delay']\n\n return df\n\n\ndef get_and_filter_group_id(df, target_group_id):\n '''\n Pluck the group_id field from request body and keep the rows equal to target_group_id.\n If target_group_id is empty, all rows are retained.\n '''\n df.group_id = px.pluck(df.req_body, 'group_id')\n df = df[px.equal(df.group_id, target_group_id) or target_group_id == '']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Kafka request.\n\n Kafka requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Kafka request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Kafka request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"consumer_group_id\",\n \"type\": \"PX_STRING\",\n \"description\": \"The group id of the consumer group.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"consumer_group_ids\",\n \"func\": {\n \"name\": \"kafka_group_ids\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n }\n },\n {\n \"outputName\": \"consumer_rebalancing_events\",\n \"func\": {\n \"name\": \"kafka_join_sync_group_events\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"consumer_group_id\",\n \"variable\": \"consumer_group_id\"\n }\n ]\n }\n },\n {\n \"outputName\": \"consumer_rebalancing_delay\",\n \"func\": {\n \"name\": \"kafka_join_sync_delay\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"consumer_group_id\",\n \"variable\": \"consumer_group_id\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Consumer Groups\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 3,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"consumer_group_ids\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Consumer Rebalancing Delay\",\n \"position\": {\n \"x\": 3,\n \"y\": 0,\n \"w\": 9,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"consumer_rebalancing_delay\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Consumer Rebalancing Events\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"consumer_rebalancing_events\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Kafka Consumer Rebalancing Events","LongDoc":"Visualizes the most recent Kafka consumer rebalancing events, with delay.\n","orgID":"","hidden":false},"px/kafka_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Kafka Data Tracer\n\nShows the most recent Kafka messages in the cluster.\n'''\nimport px\n\n\ndef kafka_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n df = add_source_dest_links(df, start_time)\n df.req_cmd = px.kafka_api_key_name(df.req_cmd)\n df = df['time_', 'source', 'destination', 'remote_port', 'req_cmd',\n 'req_body', 'resp', 'latency']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Kafka request.\n\n Kafka requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Kafka request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Kafka request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"kafka_data\",\n \"func\": {\n \"name\": \"kafka_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"kafka_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Kafka messages","LongDoc":"Shows a sample of Kafka messages in the cluster.\n","orgID":"","hidden":false},"px/kafka_overview":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Kafka Overview Map\n\nShows a graph of Kafka topic-based data flow in the cluster.\n'''\n\nimport px\n\nns_per_s = 1000 * 1000 * 1000\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n# ----------------------------------------------------------------\ndef kafka_flow_graph(start_time: str, ns: px.Namespace, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter on namespace, if specified.\n df = filter_if_not_empty(df, 'namespace', ns)\n\n # Get produce and fetch records, filtered by topic.\n producer_df = get_produce_records(df, topic)\n consumer_df = get_fetch_records(df, topic)\n\n # Setting src and dest for each edge.\n producer_df.src = producer_df.client_id\n producer_df.dest = \"topic\" + \"/\" + producer_df.topic_name\n\n consumer_df.src = \"topic\" + \"/\" + consumer_df.topic_name\n consumer_df.dest = consumer_df.client_id\n\n df = producer_df.append(consumer_df)\n\n # Get throughput by adding size of message_sets. Note that this is the total size of the\n # message batches, not the total number of bytes sent or received.\n df = df.groupby(['src', 'dest']).agg(record_bytes_total=('message_size', px.sum))\n df.record_bytes_total = px.Bytes(df.record_bytes_total)\n\n # Compute time window for the query and add it as a column.\n df = add_time_window_column(df, start_time)\n\n df.record_throughput = df.record_bytes_total / df.window\n return df\n\n\ndef kafka_topics_overview(start_time: str, ns: px.Namespace, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter on namespace, if specified.\n df = filter_if_not_empty(df, 'namespace', ns)\n\n # Get produce and fetch records, filtered by topic.\n producer_df = get_produce_records(df, topic)\n consumer_df = get_fetch_records(df, topic)\n\n df = producer_df.append(consumer_df)\n\n # Count the number of unique partitions, producers, and consumers for each topic.\n num_partitions_df = count_unique(df, 'partition_idx', 'num_partitions')\n num_producers_df = count_unique(producer_df, 'client_id', 'num_producers')\n num_consumers_df = count_unique(consumer_df, 'client_id', 'num_consumers')\n\n # Get total bytes in/out through a topic\n df_bytes_in = producer_df.groupby('topic_name').agg(bytes_produced_total=('message_size', px.sum))\n df_bytes_in.bytes_produced_total = px.Bytes(df_bytes_in.bytes_produced_total)\n df_bytes_out = consumer_df.groupby('topic_name').agg(bytes_consumed_total=('message_size', px.sum))\n df_bytes_out.bytes_consumed_total = px.Bytes(df_bytes_out.bytes_consumed_total)\n\n # Join the 5 tables together.\n df = num_partitions_df.merge(num_producers_df, how='inner', left_on='topic_name', right_on='topic_name',\n suffixes=['', '_a'])\n df = df.merge(num_consumers_df, how='inner', left_on='topic_name', right_on='topic_name',\n suffixes=['', '_b'])\n df = df.merge(df_bytes_in, how='inner', left_on='topic_name', right_on='topic_name',\n suffixes=['', '_c'])\n df = df.merge(df_bytes_out, how='inner', left_on='topic_name', right_on='topic_name',\n suffixes=['', '_d'])\n df = df[['topic_name', 'num_partitions', 'num_producers', 'num_consumers',\n 'bytes_produced_total', 'bytes_consumed_total']]\n return df\n\n\ndef kafka_brokers(start_time: str, ns: px.Namespace, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter on namespace, if specified.\n df = filter_if_not_empty(df, 'namespace', ns)\n\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Get produce and fetch records, filtered by topic.\n producer_df = get_produce_records(df, topic)\n consumer_df = get_fetch_records(df, topic)\n\n # Get total number of bytes produced and consumed to and from brokers.\n df_bytes_in = producer_df.groupby('destination').agg(bytes_produced_total=('message_size', px.sum))\n df_bytes_in.bytes_produced_total = px.Bytes(df_bytes_in.bytes_produced_total)\n df_bytes_out = consumer_df.groupby('destination').agg(bytes_consumed_total=('message_size', px.sum))\n df_bytes_out.bytes_consumed_total = px.Bytes(df_bytes_out.bytes_consumed_total)\n\n # Get throughput of records produced and consumed, grouped by broker pods.\n producer_df = compute_throughput(producer_df, start_time, \"destination\")\n consumer_df = compute_throughput(consumer_df, start_time, \"destination\")\n\n df = producer_df.merge(consumer_df, how=\"inner\", left_on=\"pod\", right_on=\"pod\", suffixes=[\"_produced\", \"_consumed\"])\n\n # Rename columns.\n df.pod = df.pod_produced\n df.produce_rate = df.throughput_produced\n df.consume_rate = df.throughput_consumed\n df.produce_requests = df.throughput_total_produced\n df.fetch_requests = df.throughput_total_consumed\n\n df = df.merge(df_bytes_in, how=\"inner\", left_on=\"pod\", right_on=\"destination\")\n df = df.merge(df_bytes_out, how=\"inner\", left_on=\"pod\", right_on=\"destination\")\n\n df = df[[\"pod\", \"produce_rate\", \"produce_requests\", \"bytes_produced_total\", \"consume_rate\",\n \"fetch_requests\", \"bytes_consumed_total\"]]\n return df\n\n\ndef kafka_producers(start_time: str, ns: px.Namespace, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter on namespace, if specified.\n df = filter_if_not_empty(df, 'namespace', ns)\n\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Get produce records, filtered by topic.\n producer_df = get_produce_records(df, topic)\n\n df_bytes_in = producer_df.groupby('source').agg(bytes_produced_total=('message_size', px.sum))\n df_bytes_in.bytes_produced_total = px.Bytes(df_bytes_in.bytes_produced_total)\n\n df = compute_throughput(producer_df, start_time, \"source\")\n df.produce_rate = df.throughput\n df.produce_requests = df.throughput_total\n df = df.merge(df_bytes_in, how=\"inner\", left_on=\"pod\", right_on=\"source\")\n\n df = df[[\"pod\", \"produce_rate\", \"produce_requests\", \"bytes_produced_total\"]]\n return df\n\n\ndef kafka_consumers(start_time: str, ns: px.Namespace, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter on namespace, if specified.\n df = filter_if_not_empty(df, 'namespace', ns)\n\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Get produce and fetch records, filtered by topic.\n consumer_df = get_fetch_records(df, topic)\n\n df_bytes_out = consumer_df.groupby('source').agg(bytes_consumed_total=('message_size', px.sum))\n df_bytes_out.bytes_consumed_total = px.Bytes(df_bytes_out.bytes_consumed_total)\n\n df = compute_throughput(consumer_df, start_time, \"source\")\n df.consume_rate = df.throughput\n df.fetch_requests = df.throughput_total\n df = df.merge(df_bytes_out, how=\"inner\", left_on=\"pod\", right_on=\"source\")\n\n df = df[[\"pod\", \"consume_rate\", \"fetch_requests\", \"bytes_consumed_total\"]]\n return df\n\n\ndef kafka_pods_flow_graph(start_time: str, ns: px.Namespace, topic: str):\n\n df = px.DataFrame('kafka_events.beta', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter on namespace, if specified.\n df = filter_if_not_empty(df, 'namespace', ns)\n\n # Get produce and fetch records, filtered by topic.\n producer_df = get_produce_records(df, topic)\n consumer_df = get_fetch_records(df, topic)\n\n df = producer_df.append(consumer_df)\n\n # Filter out records with unresolved entities.\n # Some records have unresolved entities due to an initial delay in resolving endpoints.\n df = df[df.source != \"-\" and df.destination != \"-\"]\n\n # Create 10 ns bin for time_ column\n df.timestamp = px.bin(df.time_, window_ns)\n\n df = df.groupby(['timestamp', 'source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace']).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count),\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n\n df = df.groupby(['source', 'destination', 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n return df\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n# ----------------------------------------------------------------\n# Hack to get the time window for the script.\n# TODO(philkuz): Replace this with a built-in.\ndef get_time_window(start_time: str):\n ''' Converts the start_time string into a table with a single column and single row.\n The approach is hacky, and will round to roughly 1 second.\n '''\n df = px.DataFrame('process_stats', start_time=start_time)\n\n df = df.agg(\n time_min=('time_', px.min),\n time_max=('time_', px.max),\n )\n\n df.window = px.DurationNanos(df.time_max - df.time_min)\n df = df[['window']]\n\n return df\n\n\ndef add_time_window_column(df, start_time):\n tw = get_time_window(start_time)\n df = df.merge(tw, how='inner', left_on=[], right_on=[])\n return df\n\n\ndef count_unique(df, src_col, dest_col):\n '''\n Count the unique number of values in src_col and put the result in dest_col.\n '''\n df = df.groupby(['topic_name', src_col]).agg()\n df = df.groupby('topic_name').agg(tmp=(src_col, px.count))\n df[dest_col] = df['tmp']\n df = df['topic_name', dest_col]\n return df\n\n\ndef filter_if_not_empty(df, column: str, val: str):\n '''\n Filters for rows where column is equal to val. If val is \"\", all rows are retained.\n '''\n df.criterion = px.select(df[column] == val, True, val == \"\")\n df = df[df.criterion]\n df = df.drop('criterion')\n return df\n\n\ndef unnest_topics_and_partitions(df, body_field: str):\n '''\n Unnest the topics and partitions from a data frame. body_field is the target column to unnest,\n usually 'req_body' or 'resp'.\n '''\n # Get topic_name\n df.topics = px.pluck(df[body_field], 'topics')\n df = json_unnest_first5(df, 'topic', 'topics')\n df = df[df.topic != '']\n df.topic_name = px.pluck(df.topic, 'name')\n\n # Get partition_idx\n df.partitions = px.pluck(df.topic, 'partitions')\n df = json_unnest_first5(df, 'partition', 'partitions')\n df = df[df.partition != '']\n df.partition_idx = px.pluck(df.partition, 'index')\n\n # Get message_size\n df.message_set = px.pluck(df.partition, 'message_set')\n df.message_size = px.pluck(df.message_set, 'size')\n df.message_size = px.atoi(df.message_size, 0)\n return df\n\n\ndef json_unnest_first5(df, dest_col, src_col):\n ''' Unnest the first 5 values in a JSON array in the src_col, and put it in the\n dest_col.\n '''\n df0 = json_array_index(df, dest_col, src_col, 0)\n df1 = json_array_index(df, dest_col, src_col, 1)\n df2 = json_array_index(df, dest_col, src_col, 2)\n df3 = json_array_index(df, dest_col, src_col, 3)\n df4 = json_array_index(df, dest_col, src_col, 4)\n df = df0.append(df1).append(df2).append(df3).append(df4)\n return df\n\n\ndef json_array_index(df, dest_col, src_col, idx):\n df[dest_col] = px.pluck_array(df[src_col], idx)\n return df\n\n\ndef compute_throughput(df, start_time: str, group_column: str):\n '''\n Computes the throughput (number of requests) per second and throughput total after grouping by\n the group_column.\n '''\n df = df.groupby([group_column]).agg(\n throughput_total=('latency', px.count)\n )\n df = add_time_window_column(df, start_time)\n df.throughput = df.throughput_total / df.window\n\n df.pod = df[group_column]\n df = df[[\"pod\", \"throughput\", \"throughput_total\"]]\n\n return df\n\n\ndef get_produce_records(df, topic: str):\n '''\n Get all the produce records and filter by a specified topic. If topic is empty, all\n produce records are retained.\n '''\n # Produce requests have command 0.\n producer_df = df[df.req_cmd == 0]\n\n producer_df = unnest_topics_and_partitions(producer_df, 'req_body')\n\n # Filter on topic, if specified.\n producer_df = filter_if_not_empty(producer_df, 'topic_name', topic)\n\n return producer_df\n\n\ndef get_fetch_records(df, topic: str):\n '''\n Get all the fetch records and filter by a specified topic. If topic is empty, all\n fetch records are retained.\n '''\n # Fetch requests have command 1.\n consumer_df = df[df.req_cmd == 1]\n\n consumer_df = unnest_topics_and_partitions(consumer_df, 'resp')\n\n # Filter on topic, if specified.\n consumer_df = filter_if_not_empty(consumer_df, 'topic_name', topic)\n\n return consumer_df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Kafka request.\n\n Kafka requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Kafka request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Kafka request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"topic\",\n \"type\": \"PX_STRING\",\n \"description\": \"The topic to filter on\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"kafka_flow_graph\",\n \"func\": {\n \"name\": \"kafka_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n },\n {\n \"outputName\": \"kafka_topics_overview\",\n \"func\": {\n \"name\": \"kafka_topics_overview\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n },\n {\n \"outputName\": \"kafka_brokers\",\n \"func\": {\n \"name\": \"kafka_brokers\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n },\n {\n \"outputName\": \"kafka_producers\",\n \"func\": {\n \"name\": \"kafka_producers\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n },\n {\n \"outputName\": \"kafka_consumers\",\n \"func\": {\n \"name\": \"kafka_consumers\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n },\n {\n \"outputName\": \"kafka_pods_flow_graph\",\n \"func\": {\n \"name\": \"kafka_pods_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Kafka Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"kafka_flow_graph\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"src\",\n \"toColumn\": \"dest\"\n },\n \"edgeWeightColumn\": \"record_throughput\",\n \"enableDefaultHierarchy\": true,\n \"edgeHoverInfo\": [\n \"record_bytes_total\",\n \"record_throughput\"\n ],\n \"edgeLength\": 300\n }\n },\n {\n \"name\": \"Kafka Topics\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"kafka_topics_overview\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Broker Pods\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"kafka_brokers\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Producer Pods\",\n \"position\": {\n \"x\": 0,\n \"y\": 7,\n \"w\": 6,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"kafka_producers\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Consumer Pods\",\n \"position\": {\n \"x\": 6,\n \"y\": 7,\n \"w\": 6,\n \"h\": 2\n },\n \"globalFuncOutputName\": \"kafka_consumers\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Pods Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 12,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"kafka_pods_flow_graph\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"source\",\n \"toColumn\": \"destination\"\n },\n \"edgeWeightColumn\": \"request_throughput\",\n \"edgeThresholds\": {\n \"mediumThreshold\": 100000000,\n \"highThreshold\": 500000000\n },\n \"edgeHoverInfo\": [\n \"latency_p50\",\n \"latency_p90\",\n \"latency_p99\",\n \"request_throughput\",\n \"throughput_total\"\n ],\n \"edgeLength\": 500\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Kafka Overview","LongDoc":"Overview of the Kafka cluster.\n","orgID":"","hidden":false},"px/kafka_producer_consumer_latency":{"pxl":"# Copyright 2021- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Kafka Producer-Consumer Latency\n\nThis script measures the latency for a Kafka producer-consumer pair.\nLimitations: only works if producer/consumer operate on a single topic.\n'''\nimport px\n\n\ndef kafka_producers(start_time: str, namespace: str, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter by namespace and topic.\n # TODO(chengruizhe): Use Json unnest to filter on topic once it's fully supported. Same below.\n df = df[df.namespace == namespace and px.contains(df.req_body, '\"name\":\"' + topic + '\"')]\n\n df = add_source_dest_columns(df)\n df = df.groupby(['source', 'client_id', 'req_cmd', 'namespace']).agg()\n df = df[df.req_cmd == 0]\n df.producer = df.client_id\n return df[['producer', 'source']]\n\n\ndef kafka_consumers(start_time: str, namespace: str, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n\n # Filter by namespace and topic.\n df = df[df.namespace == namespace and px.contains(df.req_body, '\"name\":\"' + topic + '\"')]\n\n df = add_source_dest_columns(df)\n df = df.groupby(['source', 'client_id', 'req_cmd', 'namespace']).agg()\n df = df[df.req_cmd == 1]\n df.consumer = df.client_id\n return df[['consumer', 'source']]\n\n\ndef kafka_topics(start_time: str, namespace: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n df.namespace = df.ctx['namespace']\n df = df[df.namespace == namespace]\n\n # Search for topics within Produce and Fetch commands.\n df = df[df.req_cmd == 0 or df.req_cmd == 1]\n df = df[['req_body']]\n\n # Unnest topics.\n df.topics = px.pluck(df.req_body, 'topics')\n df = json_unnest_first5(df, 'topic', 'topics', ['topic'])\n df = df[df.topic != '']\n df.topic = px.pluck(df.topic, 'name')\n\n df = df.groupby(['topic']).agg()\n return df\n\n\ndef kafka_data(start_time: str, namespace: str, producer: str, consumer: str, topic: str):\n df = px.DataFrame(table='kafka_events.beta', start_time=start_time)\n\n df.namespace = df.ctx['namespace']\n df.node = df.ctx['node']\n df.pod = df.ctx['pod']\n df.pid = px.upid_to_pid(df.upid)\n\n df = df[df.namespace == namespace]\n\n # Filter on the requests to the kafka topic.\n # This needs to be made more robust once PxL has better JSON querying support.\n topic_str = '\"name\":\"' + topic + '\"'\n df = df[px.contains(df.req_body, topic_str) or px.contains(df.resp, topic_str)]\n\n # Produce requests have command 0 and fetch requests have command 1.\n producer_df = df[df.req_cmd == 0]\n consumer_df = df[df.req_cmd == 1]\n\n # Filter to consumer-producer pair of interest. If consumer/producer is empty, all rows are retained.\n producer_df = producer_df[px.contains(producer_df.client_id, producer)]\n consumer_df = consumer_df[px.contains(consumer_df.client_id, consumer)]\n\n consumer_df = extract_partition_offset(consumer_df, topic, 'req_body', 'fetch_offset')\n producer_df = extract_partition_offset(producer_df, topic, 'resp', 'base_offset')\n\n df = merge_dfs(consumer_df, producer_df)\n\n df = format_label(df, consumer, producer)\n df = df[['series_col', 'time_', 'delay']]\n return df\n\n\ndef format_label(df, consumer: str, producer: str):\n df.add_producer_id = producer == \"\"\n df.add_consumer_id = consumer == \"\"\n\n df.client_id_producer = px.select(df.add_producer_id, df.client_id_producer + '/', '')\n df.client_id_consumer = px.select(df.add_consumer_id, df.client_id_consumer + '/', '')\n\n df.part_idx_consumer = 'partition-' + df.part_idx_consumer\n df.series_col = df.client_id_producer + df.client_id_consumer + df.part_idx_consumer\n return df\n\n\ndef merge_dfs(consumer_df, producer_df):\n # Self-join to match consumer requests with producer requests.\n df = consumer_df.merge(\n producer_df,\n how='inner',\n left_on=['part_idx', 'offset'],\n right_on=['part_idx', 'offset'],\n suffixes=['_consumer', '_producer'])\n\n # Compute producer consumer latency.\n # If the consumer's fetch happened before the produce, then set latency to 0,\n # since it means the consumer is ready waiting for the produce as soon as it arrives.\n df.delay = (df.time__consumer - df.time__producer) / 1000.0 / 1000.0 / 1000.0\n df.delay = px.select(df.delay \u003c 0.0, 0.0, df.delay)\n\n # Add time_ as x-axis for charting\n df.time_ = df.time__consumer\n return df\n\n\ndef extract_partition_offset(df, topic, body_field, offset_field):\n df = df[[body_field, 'client_id', 'time_']]\n\n # Unnest topics.\n df.topics = px.pluck(df[body_field], 'topics')\n df = json_unnest_first5(df, 'topic', 'topics', ['topic', 'client_id', 'time_'])\n df = df[df.topic != '']\n\n # Keep only the topic we want.\n df.topic_name = px.pluck(df.topic, 'name')\n df = df[df.topic_name == topic]\n\n # Unnest partitions.\n df.partitions = px.pluck(df.topic, 'partitions')\n df = json_unnest_first5(df, 'partition', 'partitions', ['partition', 'client_id', 'time_'])\n\n # Get index, offset, and time\n df.part_idx = px.pluck(df.partition, 'index')\n df.offset = px.pluck(df.partition, offset_field)\n df = df[['client_id', 'part_idx', 'offset', 'time_']]\n df = df[df.offset != '' and df.part_idx != '']\n return df\n\n\ndef json_unnest_first5(df, dest_col, src_col, fields):\n ''' Unnest the first 5 values in a JSON array in the src_col, and put it in the\n dest_col. Fields are the columns to keep in the resulting table.\n '''\n df0 = json_array_index(df, dest_col, src_col, fields, 0)\n df1 = json_array_index(df, dest_col, src_col, fields, 1)\n df2 = json_array_index(df, dest_col, src_col, fields, 2)\n df3 = json_array_index(df, dest_col, src_col, fields, 3)\n df4 = json_array_index(df, dest_col, src_col, fields, 4)\n df = df0.append(df1).append(df2).append(df3).append(df4)\n return df\n\n\ndef json_array_index(df, dest_col, src_col, fields, idx):\n df[dest_col] = px.pluck_array(df[src_col], idx)\n df = df[fields]\n return df\n\n\n# This needs to be re-written once PxL has better JSON querying support.\ndef extract_json_field_value(df, dest_col, src_col, field_name):\n len = px.length(field_name)\n df[dest_col] = px.substring(df[src_col], px.find(df[src_col], field_name), 100)\n df[dest_col] = px.substring(df[dest_col], len, px.find(df[dest_col], ',') - len)\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Kafka request.\n\n Kafka requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Kafka request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Kafka request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '' and df.source != '-']\n df = df[df.destination != '' and df.destination != '-']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"Namespace of the application containing kafka.\"\n },\n {\n \"name\": \"topic\",\n \"type\": \"PX_STRING\",\n \"description\": \"Kafka topic.\",\n \"defaultValue\": \"Please fill in from Kafka Topics below.\"\n },\n {\n \"name\": \"producer\",\n \"type\": \"PX_STRING\",\n \"description\": \"Kafka producer.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"consumer\",\n \"type\": \"PX_STRING\",\n \"description\": \"Kafka consumer.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"kafka_data\",\n \"func\": {\n \"name\": \"kafka_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"producer\",\n \"variable\": \"producer\"\n },\n {\n \"name\": \"consumer\",\n \"variable\": \"consumer\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Kafka Topics\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 2\n },\n \"func\": {\n \"name\": \"kafka_topics\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Producers\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 2\n },\n \"func\": {\n \"name\": \"kafka_producers\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Consumers\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 2\n },\n \"func\": {\n \"name\": \"kafka_consumers\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"topic\",\n \"variable\": \"topic\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Kafka Producer-Consumer Delay (seconds)\",\n \"globalFuncOutputName\": \"kafka_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"delay\",\n \"series\": \"series_col\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Delay (seconds)\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 2,\n \"w\": 12,\n \"h\": 3\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Kafka producer-consumer latency","LongDoc":"Shows the producer-consumer latency for a given topic. Values above 0 indicate that the consumer is falling behind the producer.\n","orgID":"","hidden":false},"px/largest_http_request":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n###############################################################\n# Script filters the largest response from the filter you pass in.\n###############################################################\n# Pods/services are formatted as \u003cnamespace\u003e/\u003cname\u003e.\n# If you want to match a namespace, only keep the namespace portion\nmatch_name = ''\nk8s_object = 'pod'\n\n# Visualization Variables - Dont change unless you know what you are doing\nfilter_dash = True\nfilter_health = True\nfilter_readyz = True\nfilter_empty_k8s = True\nresp_size = 'resp_size_bytes'\nip = 'remote_addr'\n###############################################################\n\n\ndef get_max_elm(df, column):\n ''' Returns a df where only the row with the max element in `column`'''\n max_value_df = df.agg(__max_size=(column, px.max))\n return df.merge(max_value_df, how='inner', left_on=[column], right_on=['__max_size'],\n suffixes=['', '_x']).drop('__max_size')\n\n\ndf = px.DataFrame(table='http_events', start_time='-2m')\n\ndf[resp_size] = px.Bytes(df.resp_body_size)\ndf[k8s_object] = df.ctx[k8s_object]\nfilter_pods = px.contains(df[k8s_object], match_name)\nfilter_out_conds = ((df.req_path != '/healthz' or not filter_health) and (\n df.req_path != '/readyz' or not filter_readyz)) and (\n df[ip] != '-' or not filter_dash)\nfilt_df = df[(filter_pods and filter_out_conds)]\n\nmax_size_df = get_max_elm(filt_df, resp_size)\npx.display(max_size_df[[\n k8s_object,\n resp_size,\n 'resp_body',\n 'req_path']].head(5), 'requests_of_max_size')\nqa = max_size_df.groupby(\n [k8s_object, 'req_path', resp_size]).agg(num_requests=(resp_size, px.count))\npx.display(qa, 'number of reqs')\n","vis":"","placement":"","ShortDoc":"Largest HTTP Request","LongDoc":"Calculates the largest HTTP Request according to the passed in filter value.","orgID":"","hidden":false},"px/mongodb_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' MongoDB Data Tracer\n\nShows the most recent MongoDB messages in the cluster.\n'''\nimport px\n\n\ndef mongodb_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='mongodb_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n df = add_source_dest_links(df, start_time)\n df = df[['time_', 'source', 'destination', 'req_cmd', 'req_body', 'resp_status', 'resp_body', 'latency']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the MongoDB request.\n\n MongoDB requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the MongoDB request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the MongoDB request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"mongodb_data\",\n \"func\": {\n \"name\": \"mongodb_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"mongodb_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"MongoDB Data","LongDoc":"Shows the most recent MongoDB messages in the cluster.","orgID":"","hidden":false},"px/most_http_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n###############################################################\n# Script returns the (pod, request_path) pair that has the most data\n# flowing from it.\n###############################################################\n# Pods/services are formatted as \u003cnamespace\u003e/\u003cname\u003e.\n# If you want to match a namespace, only keep the namespace portion\nmatch_name = ''\nk8s_object = 'pod'\n\n# Visualization Variables - Dont change unless you know what you are doing\nfilter_dash = True\nfilter_health = True\nfilter_readyz = True\nfilter_empty_k8s = True\nresp_size = 'resp_size_bytes'\nip = 'remote_addr'\n###############################################################\n\n\ndef get_max_elm(df, column):\n ''' Returns a df where only the row with the max element in `column`'''\n max_value_df = df.agg(__max_size=(column, px.max))\n return df.merge(max_value_df, how='inner', left_on=[column], right_on=['__max_size'],\n suffixes=['', '_x']).drop('__max_size')\n\n\ndf = px.DataFrame(table='http_events', start_time='-2m')\n\ndf[resp_size] = px.Bytes(df.resp_body_size)\ndf[k8s_object] = df.ctx[k8s_object]\nfilter_pods = px.contains(df[k8s_object], match_name)\nfilter_out_conds = ((df.req_path != '/healthz' or not filter_health) and (\n df.req_path != '/readyz' or not filter_readyz)) and (\n df[ip] != '-' or not filter_dash)\nfilt_df = df[(filter_pods and filter_out_conds)]\nk8s_and_req_path_bytes = filt_df.groupby([k8s_object, 'req_path']).agg(\n resp_bytes_sum=(resp_size, px.sum)\n)\n# Enable the following line if you want to see the bytes for every (k8s_object, 'req_path') pair\n# px.display(k8s_and_req_path_bytes, 'bytes per path')\npx.display(get_max_elm(k8s_and_req_path_bytes, 'resp_bytes_sum'), 'max_path_pod')\n","vis":"","placement":"","ShortDoc":"Pod, Endpoint pair with most HTTP Data Transferred","LongDoc":"Finds the endpoint on a specific Pod that passes the most HTTP Data. Optionally, you can uncomment a line to see a table summarizing data per service, endpoint pair.\n","orgID":"","hidden":false},"px/mux_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Mux Data Tracer\n\nThis script displays all mux traces in the cluster.\n'''\nimport px\n\n\ndef mux_data(start_time: str, num_head: int):\n\n df = px.DataFrame(table='mux_events', start_time=start_time)\n\n # Add context.\n df.node = df.ctx['node']\n df.pid = px.upid_to_pid(df.upid)\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Add mux frame type\n df.req_name = px.mux_frame_type_name(df.req_type)\n\n # Restrict number of results.\n df = df.head(num_head)\n\n # Order columns.\n df = df['time_', 'source', 'destination', 'latency', 'req_name']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the mux request.\n\n Mux requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the mux request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the mux request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"mux_data\",\n \"func\": {\n \"name\": \"mux_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"mux_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Mux Data","LongDoc":"Shows most recent Mux traffic in the cluster.","orgID":"","hidden":false},"px/mysql_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' MySQL Data Tracer\n\nShows the most recent MySQL messages in the cluster.\n'''\nimport px\n\n\ndef mysql_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='mysql_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n df = add_source_dest_links(df, start_time)\n df = df[['time_', 'source', 'destination', 'remote_port', 'req_cmd',\n 'req_body', 'resp_status', 'resp_body', 'latency']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the MySQL request.\n\n MySQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the MySQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the MySQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"mysql_data\",\n \"func\": {\n \"name\": \"mysql_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"mysql_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"MySQL Data","LongDoc":"Shows most recent MySQL messages in the cluster.","orgID":"","hidden":false},"px/mysql_flow_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' MySQL Overview Map\n\nShows a graph of MySQL requests in the cluster, with some latency information.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef mysql_flow_graph(start_time: str, ns: px.Namespace, source_filter: str,\n destination_filter: str):\n\n df = px.DataFrame('mysql_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter on namespace as specified by the user.\n df = df[df.namespace == ns]\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Create 10 ns bin for time_ column\n df.timestamp = px.bin(df.time_, window_ns)\n\n df = df.groupby(['timestamp', 'source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace']).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count),\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n\n df = df.groupby(['source', 'destination', 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n return df\n\n\ndef mysql_summary_with_links(start_time: str, ns: px.Namespace, source_filter: str,\n destination_filter: str):\n\n df = mysql_flow_graph(start_time, ns, source_filter, destination_filter)\n df = add_source_dest_links(df, start_time)\n df = df[['source', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput', 'throughput_total']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the MySQL request.\n\n MySQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the MySQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the MySQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"mysql_flow\",\n \"func\": {\n \"name\": \"mysql_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"mysql_summary_with_links\",\n \"func\": {\n \"name\": \"mysql_summary_with_links\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"MySQL Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"mysql_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"source\",\n \"toColumn\": \"destination\"\n },\n \"edgeWeightColumn\": \"request_throughput\",\n \"edgeColorColumn\": \"latency_p90\",\n \"edgeThresholds\": {\n \"mediumThreshold\": 100000000,\n \"highThreshold\": 500000000\n },\n \"edgeHoverInfo\": [\n \"latency_p50\",\n \"latency_p90\",\n \"latency_p99\",\n \"request_throughput\",\n \"throughput_total\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"mysql_summary_with_links\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"MySQL Flow Graph","LongDoc":"Graph of MySQL messages in the cluster, with latency stats.\n","orgID":"","hidden":false},"px/mysql_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' MySQL Pod LET metrics\n\nThis live view calculates the latency, error rate, and throughput\nfor each connection to a MySQL database pod.\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# None response code returns on close() of MySQL connection.\nnone_response_code = 1\n# Flag to filter out MySQL responses where no data is returned.\nfilter_responses_with_none_code = True\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(5 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization specification, vis.json.\n# ----------------------------------------------------------------\ndef pod_mysql_let(start_time: str, pod: px.Pod):\n ''' Calculate LET time-series for MySQL traffic per connection to\n a MySQL database pod.\n\n Calculates latency, error_rate, and throughput for each pod's\n connection to a MySQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor MySQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for MySQL\n traffic to a MySQL database pod.\n '''\n df = mysql_let_per_pod(start_time, pod, ['timestamp', 'destination'])\n\n return df[['time_', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'error_rate', 'request_throughput']]\n\n\ndef summary_mysql_let(start_time: str, pod: px.Pod):\n ''' Calculate LET summary for MySQL traffic per connection to\n a MySQL database pod.\n\n Calculates latency, error_rate, and throughput for each\n connection to a MySQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor MySQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for MySQL\n traffic to a MySQL database pod.\n '''\n\n df = mysql_let_per_pod(start_time, pod, ['timestamp', 'source', 'destination',\n 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace'])\n summary_df = summarize_LET(df, ['source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace'])\n summary_df_links = add_source_dest_links(summary_df, start_time)\n\n return summary_df_links[['source', 'destination', 'request_throughput', 'error_rate',\n 'latency', 'total_requests']]\n\n\ndef latency_histogram(start_time: str, pod: px.Pod):\n ''' Computes a histogram of MySQL request latency.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc.\n\n Returns: DataFrame of the MySQL latency histogram for svcs that\n match @svc.\n '''\n # The data necessary to compute MySQL LET information is located in the\n # mysql_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='mysql_events', start_time=start_time)\n df = format_mysql_table(df, filter_responses_with_none_code)\n\n # Calculate LET of pod(s) (k8s_object) connection to MySQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n matching_df.request_latency = px.DurationNanos(px.bin(matching_df.latency,\n latency_bin_size_ns))\n return matching_df.groupby('request_latency').agg(count=('time_', px.count))\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef mysql_let_per_pod(start_time: str, pod: px.Pod, groups):\n ''' Calculate LET time-series for MySQL traffic per connection to\n a MySQL database pod.\n\n Calculates latency, error_rate, and throughput for each connection to a\n MySQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor MySQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for MySQL\n traffic to a MySQL database pod.\n '''\n # The data necessary to compute MySQL LET information is located in the\n # mysql_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='mysql_events', start_time=start_time)\n df = format_mysql_table(df, filter_responses_with_none_code)\n\n # Calculate LET of pod(s) (k8s_object) connection to MySQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n let_df = calc_mysql_LET(matching_df, groups)\n\n return let_df\n\n\ndef format_events_table(df, latency_col):\n ''' Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency_ms', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on 'mysql_events' and 'http_events'\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n '''\n df.latency = df[latency_col]\n df.timestamp = px.bin(df.time_, window_ns)\n df = df[df['pod'] != '']\n\n return df\n\n\ndef format_mysql_table(df, filter_responses_with_none_code):\n ''' Formats mysql_events tables\n\n Runs events table universal formatting, creates a failure field\n marking which requests receive an error status code, and\n optionally filters out responses with \"None\" response\n error code if @filter_responses_with_none_code is true.\n\n Args:\n @df: the input mysql_events table.\n @filter_responses_with_none_code: flag to filter out MySQL\n responses where no data is returned.\n Returns: formatted mysql_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n df.failure = df['resp_status'] == 3\n\n df = df[df.resp_status\n != none_response_code or not filter_responses_with_none_code]\n return df\n\n\ndef format_LET_aggs(df):\n ''' Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics that\n can be visualized. Latency quantile values need to be extracted from the\n quantiles struct, and then error_rate and request_throughput are calculated as\n a function of window size.\n\n\n This function represents logic shared by LET calculators for MySQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total', 'error_rate_per_window', and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n '''\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n df.error_rate = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n\n return df\n\n\ndef calc_mysql_LET(df, groups):\n ''' Calculates Latency, Error Rate, and Throughput on MySQL events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups.\n\n Args:\n @df: the input mysql_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n '''\n # All requests for errors and throughput\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef summarize_LET(let_df, groups):\n ''' Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n '''\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n error_rate=('error_rate', px.mean),\n total_requests=('throughput_total', px.sum),\n latency=('latency_p50', px.mean)\n )\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the MySQL request.\n\n MySQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the MySQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the MySQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to filter by for MySQL request. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_mysql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P99 Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P99 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Error Rate\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Error Rate\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Latency Histogram\",\n \"func\": {\n \"name\": \"latency_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.HistogramChart\",\n \"histogram\": {\n \"value\": \"request_latency\",\n \"prebinCount\": \"count\",\n \"maxbins\": 10,\n \"minstep\": 5000000\n },\n \"xAxis\": {\n \"label\": \"Request Latency\"\n },\n \"yAxis\": {\n \"label\": \"# of requests\"\n }\n },\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_mysql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"MySQL Pod LET metrics","LongDoc":"This live view calculates the latency, error rate, and throughput of a pod's MySQL requests.\n","orgID":"","hidden":false},"px/namespace":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Namespace Overview\n\nThis view gives a top-level summary of the pods and services in a given namespace,\nas well as a service map.\n\n'''\nimport px\n\n\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Whether or not to include traffic from IPs that don't resolve to a known pod/service.\ninclude_ips = True\n\n\ndef pods_for_namespace(start_time: str, namespace: px.Namespace):\n ''' Gets a list of pods running per node.\n\n Args:\n @start_time Starting time of the data to examine.\n @namespace: The namespace to filter on.\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['namespace'] == namespace]\n df.pod = df.ctx['pod_name']\n df = df.groupby(['pod']).agg(\n rss=('rss_bytes', px.mean),\n vsize=('vsize_bytes', px.mean),\n )\n\n df.create_time = px.pod_name_to_start_time(df.pod)\n df.status = px.pod_name_to_status(df.pod)\n return df\n\n\ndef services_for_namespace(start_time: str, namespace: px.Namespace):\n ''' Get an overview of the services in the current cluster.\n Args:\n @start_time: The timestamp of data to start at.\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['namespace'] == namespace]\n df.service = df.ctx['service']\n df = df[df.service != '']\n df.pod = df.ctx['pod']\n df = df.groupby(['service', 'pod']).agg()\n df = df.groupby('service').agg(pod_count=('pod', px.count))\n service_let = inbound_service_let_summary(start_time, namespace)\n joined = df.merge(service_let, how='left', left_on='service', right_on='service',\n suffixes=['', '_x'])\n return joined.drop('service_x')\n\n\ndef inbound_service_let_summary(start_time: str, namespace: px.Namespace):\n ''' Compute a summary of traffic by requesting service, for requests on services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = inbound_service_let_helper(start_time, namespace)\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.trace_role == 2]\n df = df[df.service != '']\n df.responder = df.service\n per_ns_df = df.groupby(['timestamp', 'service']).agg(\n http_throughput_total=('latency', px.count),\n inbound_http_bytes_total=('req_body_size', px.sum),\n outbound_http_bytes_total=('resp_body_size', px.sum)\n )\n per_ns_df.http_request_throughput = per_ns_df.http_throughput_total / window_ns\n per_ns_df.inbound_http_throughput = per_ns_df.inbound_http_bytes_total / window_ns\n per_ns_df.outbound_http_throughput = per_ns_df.outbound_http_bytes_total / window_ns\n per_ns_df = per_ns_df.groupby('service').agg(\n http_request_throughput=('http_request_throughput', px.mean),\n inbound_http_throughput=('inbound_http_throughput', px.mean),\n outbound_http_throughput=('outbound_http_throughput', px.mean)\n )\n quantiles_df = df.groupby('service').agg(\n http_latency=('latency', px.quantiles)\n http_error_rate=('failure', px.mean),\n )\n quantiles_df.http_error_rate = px.Percent(quantiles_df.http_error_rate)\n joined = per_ns_df.merge(quantiles_df, left_on='service',\n right_on='service', how='inner',\n suffixes=['', '_x'])\n return joined[['service', 'http_latency', 'http_request_throughput', 'http_error_rate',\n 'inbound_http_throughput', 'outbound_http_throughput']]\n\n\ndef inbound_service_let_graph(start_time: str, namespace: px.Namespace):\n ''' Compute a summary of traffic by requesting service, for requests on services in `namespace`.\n Similar to `inbound_let_summary` but also breaks down by pod in addition to service.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = inbound_service_let_helper(start_time, namespace)\n df = df.groupby(['timestamp', 'service', 'remote_addr', 'pod', 'trace_role']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate=('failure', px.mean),\n throughput_total=('latency', px.count),\n inbound_bytes_total=('req_body_size', px.sum),\n outbound_bytes_total=('resp_body_size', px.sum)\n )\n\n # Get the traced and remote pod/service/IP information.\n df.traced_pod = df.pod\n df.traced_service = df.service\n df.traced_ip = px.pod_name_to_pod_ip(df.pod)\n\n localhost_ip_regexp = r'127\\.0\\.0\\.[0-9]+'\n df.is_remote_addr_localhost = px.regex_match(localhost_ip_regexp, df.remote_addr)\n df.remote_pod = px.select(df.is_remote_addr_localhost,\n df.pod,\n px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr)))\n df.remote_service = px.select(df.is_remote_addr_localhost,\n df.traced_service,\n px.service_id_to_service_name(px.ip_to_service_id(df.remote_addr)))\n\n df.remote_ip = df.remote_addr\n # If external IPs are excluded in the service graph, then we also exclude any\n # traffic where we don't know the remote pod or remote service name.\n df = df[include_ips or (df.remote_pod != '' or df.remote_service != '')]\n\n # Associate it with Client/Server roles, based on the trace role.\n df.is_server_side_tracing = df.trace_role == 2\n df.responder_pod = px.select(df.is_server_side_tracing, df.traced_pod, df.remote_pod)\n df.requestor_pod = px.select(df.is_server_side_tracing, df.remote_pod, df.traced_pod)\n df.responder_service = px.select(df.is_server_side_tracing, df.traced_service, df.remote_service)\n df.requestor_service = px.select(df.is_server_side_tracing, df.remote_service, df.traced_service)\n df.responder_ip = px.select(df.is_server_side_tracing, df.traced_ip, df.remote_ip)\n df.requestor_ip = px.select(df.is_server_side_tracing, df.remote_ip, df.traced_ip)\n\n # Compute statistics about each edge of the service graph.\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n df.inbound_throughput = df.inbound_bytes_total / window_ns\n df.outbound_throughput = df.outbound_bytes_total / window_ns\n df.error_rate = px.Percent(df.error_rate)\n return df.groupby(['responder_pod', 'requestor_pod', 'responder_service',\n 'requestor_service', 'responder_ip', 'requestor_ip']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n error_rate=('error_rate', px.mean),\n inbound_throughput=('inbound_throughput', px.mean),\n outbound_throughput=('outbound_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n\ndef inbound_service_let_helper(start_time: str, namespace: px.Namespace):\n ''' Compute the let as a timeseries for requests received or by services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n df.service = df.ctx['service']\n df.pod = df.ctx['pod_name']\n df = df[df.ctx['namespace'] == namespace]\n df = df[df.pod != '']\n df.latency = df.latency\n df.timestamp = px.bin(df.time_, window_ns)\n\n df.failure = df.resp_status \u003e= 400\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n\n df = df[filter_out_conds]\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The name of the namespace.\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"HTTP Service Map\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"inbound_service_let_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.RequestGraph\",\n \"requestorPodColumn\": \"requestor_pod\",\n \"responderPodColumn\": \"responder_pod\",\n \"requestorServiceColumn\": \"requestor_service\",\n \"responderServiceColumn\": \"responder_service\",\n \"requestorIPColumn\": \"requestor_ip\",\n \"responderIPColumn\": \"responder_ip\",\n \"p50Column\": \"latency_p50\",\n \"p90Column\": \"latency_p90\",\n \"p99Column\": \"latency_p99\",\n \"errorRateColumn\": \"error_rate\",\n \"requestsPerSecondColumn\": \"request_throughput\",\n \"inboundBytesPerSecondColumn\": \"inbound_throughput\",\n \"outboundBytesPerSecondColumn\": \"outbound_throughput\",\n \"totalRequestCountColumn\": \"throughput_total\"\n }\n },\n {\n \"name\": \"Service List\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"services_for_namespace\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Pod List\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"pods_for_namespace\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Namespace Overview","LongDoc":"This view gives a top-level summary of the pods and services in a given namespace, as well as a service map.\n","orgID":"","hidden":false},"px/namespaces":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Namespaces Overview\n\nThis view lists the namespaces on the current cluster and their pod and service counts.\nIt also lists the high-level resource consumption by namespace.\n\n'''\nimport px\nimport pxviews\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef namespaces_for_cluster(start_time: str):\n ''' Gets a list of namespaces in the current cluster since `start_time`.\n\n Args:\n @start_time Start time of the data to examine.\n '''\n df = pxviews.pod_resource_stats(start_time, px.now())\n pod_count = df.groupby('namespace').agg(pod_count=('pod', px.count))\n services = df.groupby(['namespace', 'service']).agg()\n service_count = services.groupby('namespace').agg(service_count=('service', px.count))\n df = pod_count.merge(service_count, how='inner', left_on='namespace', right_on='namespace', suffixes=('', '_x'))\n return df[['namespace', 'pod_count', 'service_count']]\n\n\ndef process_stats_by_namespace(start_time: str):\n ''' Gets a summary of process stats by namespace since `start_time`.\n Computes the total I/O consumption across the namespace since `start_time`.\n\n Args:\n @start_time Start time of the data to examine.\n '''\n df = pxviews.pod_resource_stats(start_time, px.now())\n df = df.groupby('namespace').agg(\n vsize=('vsize', px.sum),\n rss=('rss', px.sum),\n actual_disk_read_throughput=('actual_disk_read_throughput', px.sum),\n actual_disk_write_throughput=('actual_disk_write_throughput', px.sum),\n total_disk_read_throughput=('total_disk_read_throughput', px.sum),\n total_disk_write_throughput=('total_disk_write_throughput', px.sum),\n )\n\n return df[[\n 'namespace',\n 'vsize',\n 'rss',\n 'actual_disk_read_throughput',\n 'actual_disk_write_throughput',\n 'total_disk_read_throughput',\n 'total_disk_write_throughput',\n ]]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Namespaces\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"namespaces_for_cluster\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Process Stats Overview by Namespace\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"process_stats_by_namespace\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Namespaces Overview","LongDoc":"This view lists the namespaces on the current cluster and their pod and service counts. It also lists the high-level resource consumption by namespace.\n","orgID":"","hidden":false},"px/nats_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' NATS Events Tracer\n\nThis script traces all NATS data in the cluster.\n\nIf you are using TLS for NATS, ensure that your NATS server and client are built\nwith debug symbols included. Otherwise, this script will produce no data.\n'''\nimport px\n\n\ndef nats_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='nats_events.beta', start_time=start_time)\n\n # Add context.\n df.pid = px.upid_to_pid(df.upid)\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n # Order columns.\n df = df['time_', 'source', 'destination', 'cmd', 'body', 'resp', 'pid']\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the NATS message.\n\n NATS messages are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the message is traced client-side (trace_role==1).\n\n When trace_role==2, the NATS message source is the remote_addr column\n and destination is the pod column. When trace_role==1, the NATS message\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"nats_data\",\n \"func\": {\n \"name\": \"nats_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"nats_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"NATS data","LongDoc":"Shows most recent NATS messages in the cluster.","orgID":"","hidden":false},"px/net_flow_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef net_flow_graph(start_time: str, ns: px.Namespace, from_entity_filter: str,\n to_entity_filter: str, throughput_filter: float):\n df = px.DataFrame('conn_stats', start_time=start_time)\n\n # Filter on namespace.\n df = df[df.ctx['namespace'] == ns]\n\n # Filter for client side requests.\n df = df[df.trace_role == 1]\n\n # Store the pod. Ideally this would be done after the aggregate,\n # but that's not working right now.\n df.pod = df.ctx['pod']\n\n # Filter out any non k8s sources.\n df = df[df.pod != '']\n\n # Find the time window\n time_window = df.agg(\n time_min=('time_', px.min),\n time_max=('time_', px.max),\n )\n time_window.time_delta = px.DurationNanos(time_window.time_max - time_window.time_min)\n time_window = time_window.drop(['time_min', 'time_max'])\n\n # Use aggregate to pick the first and last sample for any given client-server pair.\n # We do this by picking the min/max of the stats, since they are all counters.\n df = df.groupby(['pod', 'upid', 'remote_addr']).agg(\n bytes_sent_min=('bytes_sent', px.min),\n bytes_sent_max=('bytes_sent', px.max),\n bytes_recv_min=('bytes_recv', px.min),\n bytes_recv_max=('bytes_recv', px.max),\n )\n df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n df.bytes_total = df.bytes_sent + df.bytes_recv\n df = df.drop(['bytes_sent_max', 'bytes_sent_min', 'bytes_recv_max', 'bytes_recv_min'])\n\n # To create a graph, add 'from' and 'to' entities.\n df.from_entity = df.pod\n\n # TODO(yzhao): Handle IPv6 ::1 as well.\n localhost_ip_regexp = r'127\\.0\\.0\\.[0-9]+'\n df.is_remote_addr_localhost = px.regex_match(localhost_ip_regexp, df.remote_addr)\n df.to_entity = px.select(df.is_remote_addr_localhost,\n df.pod,\n px.nslookup(df.remote_addr))\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.from_entity, from_entity_filter)]\n df = df[px.contains(df.to_entity, to_entity_filter)]\n\n # Since there may be multiple processes per pod,\n # perform an additional aggregation to consolidate those into one entry.\n df = df.groupby(['from_entity', 'to_entity']).agg(\n bytes_sent=('bytes_sent', px.sum),\n bytes_recv=('bytes_recv', px.sum),\n bytes_total=('bytes_total', px.sum),\n )\n\n # Add time_delta to every row. Use a join to do this.\n # Future syntax will support: df.time_delta = time_window.at[0, 'time_delta']\n df.join_key = 1\n time_window.join_key = 1\n df = df.merge(time_window, how='inner', left_on='join_key', right_on='join_key')\n df = df.drop(['join_key_x', 'join_key_y'])\n\n # Compute as rates.\n df.bytes_sent = df.bytes_sent / df.time_delta\n df.bytes_recv = df.bytes_recv / df.time_delta\n df.bytes_total = df.bytes_total / df.time_delta\n df = df.drop(['time_delta'])\n\n # Apply rate filter.\n df = df[df.bytes_total \u003e throughput_filter / 1000000000]\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on\"\n },\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"from_entity_filter\",\n \"type\": \"PX_POD\",\n \"description\": \"The partial string to match the 'from_entity' (source pod).\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"to_entity_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'to_entity' (destination service).\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"throughput_filter\",\n \"type\": \"PX_FLOAT64\",\n \"description\": \"Filters out connections with total throughput less than the specified amount. Specified in B/sec.\",\n \"defaultValue\": \"0.0\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"net_flow\",\n \"func\": {\n \"name\": \"net_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"from_entity_filter\",\n \"variable\": \"from_entity_filter\"\n },\n {\n \"name\": \"to_entity_filter\",\n \"variable\": \"to_entity_filter\"\n },\n {\n \"name\": \"throughput_filter\",\n \"variable\": \"throughput_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Net Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"net_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"from_entity\",\n \"toColumn\": \"to_entity\"\n },\n \"edgeWeightColumn\": \"bytes_total\",\n \"edgeColorColumn\": \"bytes_total\",\n \"edgeHoverInfo\": [\n \"bytes_total\",\n \"bytes_sent\",\n \"bytes_recv\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 4,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"net_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Network Flow","LongDoc":"The mapping of all outgoing connections for the specified k8s pods in the namespace.\n","orgID":"","hidden":false},"px/network_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\nt1 = px.DataFrame(table='network_stats', select=['time_', 'pod_id', 'rx_bytes',\n 'rx_packets', 'rx_errors',\n 'rx_drops', 'tx_bytes', 'tx_packets',\n 'tx_errors', 'tx_drops'], start_time='-30s')\n\nt2 = t1.head(n=100)\n\npx.display(t2)\n","vis":"","placement":"","ShortDoc":"Network Stats","LongDoc":"Get network stats time series","orgID":"","hidden":false},"px/node":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Node Overview\n\nThis view summarizes the process and network stats for a given input node in a cluster.\nIt computes CPU, memory consumption, as well as network traffic statistics.\nIt also displays a list of pods that were on that node during the time window.\n\n'''\nimport px\nimport pxviews\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef pods_for_node(start_time: str, node: px.Node):\n ''' Gets a list of pods running on the input node.\n\n Args:\n @start_time Starting time of the data to examine.\n @node: The full name of the node to filter on.\n '''\n df = pxviews.pod_resource_stats(px.now() + px.parse_duration(start_time), px.now())\n df = df[df.ctx['node'] == node]\n df.containers = df.container_count\n df.start_time = df.pod_start_time\n df.status = df.pod_status\n return df[['pod', 'start_time', 'containers', 'status']]\n\n\ndef resource_timeseries(start_time: str, node: px.Node, groupby: str):\n ''' Gets the windowed process stats (CPU, memory, etc) for the input node.\n\n Args:\n @start_time Starting time of the data to examine.\n @node: The full name of the node to filter on.\n '''\n df = pxviews.container_process_timeseries(start_time, px.now(), window_ns)\n df = df[df.ctx['node'] == node]\n df[groupby] = df.ctx[groupby]\n\n df = df.groupby(['time_', groupby]).agg(\n cpu_usage=('cpu_usage', px.sum),\n actual_disk_read_throughput=('actual_disk_read_throughput', px.sum),\n actual_disk_write_throughput=('actual_disk_write_throughput', px.sum),\n total_disk_read_throughput=('total_disk_read_throughput', px.sum),\n total_disk_write_throughput=('total_disk_write_throughput', px.sum),\n rss=('rss', px.sum),\n vsize=('vsize', px.sum),\n )\n\n df.groupby_col = df[groupby]\n return df\n\n\ndef network_stats(start_time: str, node: px.Node, groupby: str):\n ''' Gets the network stats (transmitted/received traffic) for the input node.\n\n Args:\n @start_time Starting time of the data to examine.\n @node: The full name of the node to filter on.\n '''\n df = pxviews.pod_network_timeseries(start_time, px.now(), window_ns)\n df = df[df.ctx['node'] == node]\n df.groupby_col = df.ctx[groupby]\n\n # Add up the network values per node.\n df = df.groupby(['time_', 'groupby_col']).agg(\n rx_bytes_per_ns=('rx_bytes_per_ns', px.sum),\n tx_bytes_per_ns=('tx_bytes_per_ns', px.sum),\n rx_drops_per_ns=('rx_drops_per_ns', px.sum),\n tx_drops_per_ns=('tx_drops_per_ns', px.sum),\n rx_errors_per_ns=('rx_errors_per_ns', px.sum),\n tx_errors_per_ns=('tx_errors_per_ns', px.sum),\n )\n return df[['time_', 'groupby_col', 'rx_bytes_per_ns', 'tx_bytes_per_ns',\n 'rx_drops_per_ns', 'tx_drops_per_ns', 'rx_errors_per_ns', 'tx_errors_per_ns']]\n\n\ndef stacktraces(start_time: str, node: str):\n df = pxviews.stacktraces(start_time, px.now())\n\n # Apply filters.\n df = df[df.node == node]\n\n grouping_agg = df.groupby(['node']).agg(\n node_count_sum=('count', px.sum),\n )\n\n # Filter out any non-k8s processes (it's now safe to do so).\n df = df[df.pod != '']\n\n # Compute percentages.\n df = df.merge(\n grouping_agg,\n how='inner',\n left_on='node',\n right_on='node',\n suffixes=['', '_x']\n )\n\n df.percent = 100.0 * df.count * df.node_num_cpus / df.node_count_sum\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"node\",\n \"type\": \"PX_NODE\",\n \"description\": \"The node name to filter on\"\n },\n {\n \"name\": \"groupby\",\n \"type\": \"PX_STRING\",\n \"description\": \"The element to groupby\",\n \"defaultValue\": \"node\",\n \"validValues\": [\n \"node\",\n \"service\",\n \"pod\",\n \"namespace\"\n ]\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"resource_timeseries\",\n \"func\": {\n \"name\": \"resource_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node\",\n \"variable\": \"node\"\n },\n {\n \"name\": \"groupby\",\n \"variable\": \"groupby\"\n }\n ]\n }\n },\n {\n \"outputName\": \"network_stats\",\n \"func\": {\n \"name\": \"network_stats\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node\",\n \"variable\": \"node\"\n },\n {\n \"name\": \"groupby\",\n \"variable\": \"groupby\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Pods\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 8,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"pods_for_node\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node\",\n \"variable\": \"node\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n }\n },\n {\n \"name\": \"CPU Usage\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"cpu_usage\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"CPU usage\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Bytes read\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_read_throughput\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Bytes read\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Bytes written\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_write_throughput\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Bytes written\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Sent Network Traffic\",\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"network_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"tx_bytes_per_ns\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Sent Data\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Received Network Traffic\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"network_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"rx_bytes_per_ns\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Received Data\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Resident Set Size\",\n \"position\": {\n \"x\": 4,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"rss\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Resident Memory Usage\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Virtual Memory Size\",\n \"position\": {\n \"x\": 8,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"vsize\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"groupby_col\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Virtual Memory Usage\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"CPU Flamegraph\",\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 12,\n \"h\": 5\n },\n \"func\": {\n \"name\": \"stacktraces\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node\",\n \"variable\": \"node\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.StackTraceFlameGraph\",\n \"stacktraceColumn\": \"stack_trace\",\n \"countColumn\": \"count\",\n \"percentageColumn\": \"percent\",\n \"namespaceColumn\": \"namespace\",\n \"podColumn\": \"pod\",\n \"containerColumn\": \"container\",\n \"pidColumn\": \"cmdline\",\n \"percentageLabel\": \"CPU Usage\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Node overview","LongDoc":"This view summarizes the process and network stats for a given input node in a cluster. It computes CPU, memory consumption, as well as network traffic statistics. It also displays a list of pods that were on that node during the time window.\n","orgID":"","hidden":false},"px/nodes":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Nodes Overview\n\nThis view summarizes the process and network stats for each node in a cluster.\nIt computes CPU, memory consumption, as well as network traffic statistics, per node.\nIt also displays a list of pods that were on each node during the time window.\n\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef nodes(start_time: str):\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df.node = df.ctx['node_name']\n return df.groupby('node').agg()\n\n\ndef pods_by_node(start_time: str):\n ''' Gets a list of pods running per node.\n\n Args:\n @start_time Starting time of the data to examine.\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df.node = df.ctx['node_name']\n df.pod = df.ctx['pod_name']\n df = df.groupby(['node', 'pod']).agg()\n df.pod_create_time = px.pod_name_to_start_time(df.pod)\n df.pod_status = px.pod_name_to_status(df.pod)\n return df\n\n\ndef process_stats(start_time: str):\n ''' Gets the windowed process stats (CPU, memory, etc) per node.\n\n Args:\n @start_time Starting time of the data to examine.\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df.node = df.ctx['node_name']\n df.timestamp = px.bin(df.time_, window_ns)\n\n # First calculate CPU usage by process (UPID) in each k8s_object\n # over all windows.\n df = df.groupby(['node', 'upid', 'timestamp']).agg(\n rss=('rss_bytes', px.mean),\n vsize=('vsize_bytes', px.mean),\n # The fields below are counters, so we take the min and the max to subtract them.\n cpu_utime_ns_max=('cpu_utime_ns', px.max),\n cpu_utime_ns_min=('cpu_utime_ns', px.min),\n cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n read_bytes_max=('read_bytes', px.max),\n read_bytes_min=('read_bytes', px.min),\n write_bytes_max=('write_bytes', px.max),\n write_bytes_min=('write_bytes', px.min),\n rchar_bytes_max=('rchar_bytes', px.max),\n rchar_bytes_min=('rchar_bytes', px.min),\n wchar_bytes_max=('wchar_bytes', px.max),\n wchar_bytes_min=('wchar_bytes', px.min),\n )\n\n # Next calculate cpu usage and memory stats per window.\n df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n df.actual_disk_read_throughput = (df.read_bytes_max - df.read_bytes_min) / window_ns\n df.actual_disk_write_throughput = (df.write_bytes_max - df.write_bytes_min) / window_ns\n df.total_disk_read_throughput = (df.rchar_bytes_max - df.rchar_bytes_min) / window_ns\n df.total_disk_write_throughput = (df.wchar_bytes_max - df.wchar_bytes_min) / window_ns\n\n # Then aggregate process individual process metrics.\n df = df.groupby(['node', 'timestamp']).agg(\n cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n cpu_utime_ns=('cpu_utime_ns', px.sum),\n actual_disk_read_throughput=('actual_disk_read_throughput', px.sum),\n actual_disk_write_throughput=('actual_disk_write_throughput', px.sum),\n total_disk_read_throughput=('total_disk_read_throughput', px.sum),\n total_disk_write_throughput=('total_disk_write_throughput', px.sum),\n rss=('rss', px.sum),\n vsize=('vsize', px.sum),\n )\n\n # Finally, calculate total (kernel + user time) percentage used over window.\n df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n df['time_'] = df['timestamp']\n return df.drop(['cpu_ktime_ns', 'cpu_utime_ns', 'timestamp'])\n\n\ndef network_stats(start_time: str):\n ''' Gets the network stats (transmitted/received traffic) per node.\n\n Args:\n @start_time Starting time of the data to examine.\n '''\n df = px.DataFrame(table='network_stats', start_time=start_time)\n df.node = px.pod_id_to_node_name(df.pod_id)\n df.timestamp = px.bin(df.time_, window_ns)\n\n # First calculate network usage by node over all windows.\n # Data is sharded by Pod in network_stats.\n df = df.groupby(['node', 'pod_id', 'timestamp']).agg(\n rx_bytes_end=('rx_bytes', px.max),\n rx_bytes_start=('rx_bytes', px.min),\n tx_bytes_end=('tx_bytes', px.max),\n tx_bytes_start=('tx_bytes', px.min),\n tx_errors_end=('tx_errors', px.max),\n tx_errors_start=('tx_errors', px.min),\n rx_errors_end=('rx_errors', px.max),\n rx_errors_start=('rx_errors', px.min),\n tx_drops_end=('tx_drops', px.max),\n tx_drops_start=('tx_drops', px.min),\n rx_drops_end=('rx_drops', px.max),\n rx_drops_start=('rx_drops', px.min),\n )\n\n # Calculate the network statistics rate over the window.\n # We subtract the counter value at the beginning ('_start')\n # from the value at the end ('_end').\n df.rx_bytes_per_ns = (df.rx_bytes_end - df.rx_bytes_start) / window_ns\n df.tx_bytes_per_ns = (df.tx_bytes_end - df.tx_bytes_start) / window_ns\n df.rx_drops_per_ns = (df.rx_drops_end - df.rx_drops_start) / window_ns\n df.tx_drops_per_ns = (df.tx_drops_end - df.tx_drops_start) / window_ns\n df.rx_errors_per_ns = (df.rx_errors_end - df.rx_errors_start) / window_ns\n df.tx_errors_per_ns = (df.tx_errors_end - df.tx_errors_start) / window_ns\n\n # Add up the network values per node.\n df = df.groupby(['node', 'timestamp']).agg(\n rx_bytes_per_ns=('rx_bytes_per_ns', px.sum),\n tx_bytes_per_ns=('tx_bytes_per_ns', px.sum),\n rx_drop_per_ns=('rx_drops_per_ns', px.sum),\n tx_drops_per_ns=('tx_drops_per_ns', px.sum),\n rx_errors_per_ns=('rx_errors_per_ns', px.sum),\n tx_errors_per_ns=('tx_errors_per_ns', px.sum),\n )\n df['time_'] = df['timestamp']\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"process_stats\",\n \"func\": {\n \"name\": \"process_stats\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n }\n },\n {\n \"outputName\": \"network_stats\",\n \"func\": {\n \"name\": \"network_stats\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Nodes\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"nodes\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"CPU Usage\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"process_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"cpu_usage\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"node\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"CPU usage\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Sent Network Traffic\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"network_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"tx_bytes_per_ns\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"node\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Sent Data\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Received Network Traffic\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"network_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"rx_bytes_per_ns\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"node\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Received Data\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Bytes read\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"process_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_read_throughput\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"node\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Bytes read\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Bytes written\",\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"process_stats\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_write_throughput\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"node\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Bytes written\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Pods\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"pods_by_node\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Nodes overview","LongDoc":"This view summarizes the process and network stats for each node in a cluster. It computes CPU, memory consumption, as well as network traffic statistics, per node. It also displays a list of pods that were on each node during the time window.\n","orgID":"","hidden":false},"px/outbound_conns":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Outbound Connections\n\nThis script lists connections made to services that are not within the k8s cluster.\n'''\nimport px\n\n\ndef outbound_conns(start_time: str, ip_filter: str):\n df = px.DataFrame(table='conn_stats', start_time=start_time)\n\n df.namespace = df.ctx['namespace']\n df.pod = df.ctx['pod']\n\n # Filter for outbound traffic only.\n # Trace-role of 1 means client-side tracing. Pixie only traces\n # on the client side when traffic is leaving the cluster.\n df = df[df.trace_role == 1]\n\n # Filter out any client-side tracing to known pods.\n df.remote_pod_id = px.ip_to_pod_id(df.remote_addr)\n df.remote_service_id = px.ip_to_service_id(df.remote_addr)\n df = df[df.remote_pod_id == '' and df.remote_service_id == '']\n\n # Filter out connections to localhost.\n df = df[not df.remote_addr == '127.0.0.1']\n df = df[not df.remote_addr == '0.0.0.0']\n\n # Apply user supplied filter.\n df = df[px.contains(df.remote_addr, ip_filter)]\n\n # Calculate connection stats for each process for each unique pod / remote_addr pair.\n df = df.groupby(['pod', 'upid', 'remote_addr', 'remote_port']).agg(\n # The fields below are counters per UPID, so we take\n # the min (starting value) and the max (ending value) to subtract them.\n conn_open_min=('conn_open', px.min),\n conn_open_max=('conn_open', px.max),\n bytes_sent_min=('bytes_sent', px.min),\n bytes_sent_max=('bytes_sent', px.max),\n bytes_recv_min=('bytes_recv', px.min),\n bytes_recv_max=('bytes_recv', px.max),\n last_activity_time=('time_', px.max)\n )\n\n # Calculate connection stats over the time window.\n df.conn_open = df.conn_open_max - df.conn_open_min\n df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n\n # Calculate connection stats for each unique pod / remote_addr pair. Since there\n # may be multiple processes per pod we perform an additional aggregation to\n # consolidate those into one entry.\n df = df.groupby(['pod', 'remote_addr', 'remote_port']).agg(\n conn_open=('conn_open', px.sum),\n bytes_sent=('bytes_sent', px.sum),\n bytes_recv=('bytes_recv', px.sum),\n last_activity_time=('last_activity_time', px.max)\n )\n\n return df[['pod', 'remote_addr', 'remote_port', 'conn_open', 'bytes_sent', 'bytes_recv',\n 'last_activity_time']]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-24h\"\n },\n {\n \"name\": \"ip_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"IP filter, using a substring match.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"outbound_conns\",\n \"func\": {\n \"name\": \"outbound_conns\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ip_filter\",\n \"variable\": \"ip_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"outbound_conns\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Outbound Connections","LongDoc":"Shows a list of connections that are to endpoints outside the k8s cluster.","orgID":"","hidden":false},"px/perf_flamegraph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n'''CPU Flamegraph\n\nPerformance profile visualization.\n\n'''\nimport px\n\n\ndef stacktraces(start_time: str, node: str, namespace: str, pod: str, pct_basis_entity: str):\n df = px.DataFrame(table='stack_traces.beta', start_time=start_time)\n\n df.namespace = df.ctx['namespace']\n df.pod = df.ctx['pod']\n df.container = df.ctx['container']\n df.cmdline = df.ctx['cmdline']\n\n # Compute node using _exec_hostname() instead of `df.ctx['node']`\n # We do this so it works for non-k8s processes too.\n # This is important for determining total number of stack trace samples per node,\n # as we need to include the non-K8s processes in the computation.\n df.node = px.Node(px._exec_hostname())\n\n # This variable is used when later computing percentages.\n # When pct_basis_entity is pod, then the computation is simple:\n # percentage = num_samples_symbol / num_samples_pod\n # But when pct_basis_entity is node, the computation needs to account for the number of CPUs,\n # because we want CPU percentages, not Node percentages (a Node may have multiple CPUs).\n # The formula then becomes:\n # percentage = num_samples_symbol / (num_samples_node / num_cpus )\n # Note that the difference between the two formulas is that the denominator is scaled.\n # We call this extra quotient the \"scaling_factor\".\n # Also note each node may have a different number of CPUs, so we have to compute this early.\n df.scaling_factor = px.select(pct_basis_entity == 'node', px._exec_host_num_cpus(), 1)\n\n # This must be done before any filtering, to get accurate stack trace totals.\n grouping_agg = df.groupby([pct_basis_entity]).agg(\n count=('count', px.sum),\n scaling_factor=('scaling_factor', px.any)\n )\n\n # Apply filters.\n df = df[px.contains(df.node, node)]\n df = df[px.contains(df.pod, pod)]\n df = df[px.contains(df.namespace, namespace)]\n\n # Filter out any non-k8s processes (it's now safe to do so).\n df = df[df.pod != '']\n\n # Aggregate stack-traces from different profiles into one larger profile.\n # For example, if a profile is generated every 30 seconds, and our query spans 5 minutes,\n # this merges the 10 profiles into a single profile including samples for entire 5 minutes.\n df = df.groupby(['node', 'namespace', 'pod', 'container', 'cmdline', 'stack_trace_id']).agg(\n stack_trace=('stack_trace', px.any),\n count=('count', px.sum)\n )\n\n # Compute percentages.\n df = df.merge(\n grouping_agg,\n how='inner',\n left_on=pct_basis_entity,\n right_on=pct_basis_entity,\n suffixes=['', '_x']\n )\n df.percent = 100.0 * df.count * df.scaling_factor / df.count_x\n return df.drop(pct_basis_entity + '_x')\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"node\",\n \"type\": \"PX_NODE\",\n \"description\": \"The node to filter on.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The pod to filter on.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"pct_basis_entity\",\n \"type\": \"PX_STRING\",\n \"description\": \"When computing percentages, the entity to use as the quotient.\",\n \"defaultValue\": \"node\",\n \"validValues\": [\n \"node\",\n \"pod\"\n ]\n\n }\n ],\n \"globalFuncs\": [],\n \"widgets\": [\n {\n \"name\": \"Flamegraph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 6\n },\n \"func\": {\n \"name\": \"stacktraces\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"node\",\n \"variable\": \"node\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n },\n {\n \"name\": \"pct_basis_entity\",\n \"variable\": \"pct_basis_entity\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.StackTraceFlameGraph\",\n \"stacktraceColumn\": \"stack_trace\",\n \"countColumn\": \"count\",\n \"percentageColumn\": \"percent\",\n \"namespaceColumn\": \"namespace\",\n \"podColumn\": \"pod\",\n \"containerColumn\": \"container\",\n \"pidColumn\": \"cmdline\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Flamegraph","LongDoc":"Shows stack trace samples that indicate where applications are spending their time.\n","orgID":"","hidden":false},"px/pgsql_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' PostgreSQL Data Tracer\n\nShows the most recent PostgreSQL messages in the cluster.\n'''\nimport px\n\n\ndef pgsql_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='pgsql_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n df = add_source_dest_links(df, start_time)\n df = df[['time_', 'source', 'destination', 'remote_port', 'req', 'resp', 'latency']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the PostgreSQL request.\n\n PostgreSQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the PostgreSQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the PostgreSQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"pgsql_data\",\n \"func\": {\n \"name\": \"pgsql_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"pgsql_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Postgres Data","LongDoc":"Shows most recent PGSQL (Postgres) messages in the cluster.","orgID":"","hidden":false},"px/pgsql_flow_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' PostgreSQL Overview Map\n\nShows a graph of PostgreSQL requests in the cluster, with some latency information.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef pgsql_flow_graph(start_time: str, ns: px.Namespace, source_filter: str,\n destination_filter: str):\n\n df = px.DataFrame('pgsql_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter on namespace as specified by the user.\n df = df[df.namespace == ns]\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Create 10 ns bin for time_ column\n df.timestamp = px.bin(df.time_, window_ns)\n\n df = df.groupby(['timestamp', 'source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace']).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count),\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n\n df = df.groupby(['source', 'destination', 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n return df\n\n\ndef pgsql_summary_with_links(start_time: str, ns: px.Namespace, source_filter: str,\n destination_filter: str):\n\n df = pgsql_flow_graph(start_time, ns, source_filter, destination_filter)\n df = add_source_dest_links(df, start_time)\n df = df[['source', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput', 'throughput_total']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the PostgreSQL request.\n\n PostgreSQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the PostgreSQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the PostgreSQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"pgsql_flow\",\n \"func\": {\n \"name\": \"pgsql_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"pgsql_summary_with_links\",\n \"func\": {\n \"name\": \"pgsql_summary_with_links\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"PostgreSQL Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"pgsql_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"source\",\n \"toColumn\": \"destination\"\n },\n \"edgeWeightColumn\": \"request_throughput\",\n \"edgeColorColumn\": \"latency_p90\",\n \"edgeThresholds\": {\n \"mediumThreshold\": 100000000,\n \"highThreshold\": 500000000\n },\n \"edgeHoverInfo\": [\n \"latency_p50\",\n \"latency_p90\",\n \"latency_p99\",\n \"request_throughput\",\n \"throughput_total\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"pgsql_summary_with_links\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"PostgreSQL Flow Graph","LongDoc":"Graph of PostgreSQL messages in the cluster, with latency stats.\n","orgID":"","hidden":false},"px/pgsql_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' PostgreSQL Pod LET metrics\n\nThis live view calculates the latency, error rate, and throughput\nfor each connection to a PostgreSQL database pod.\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(50 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization specification, vis.json.\n# ----------------------------------------------------------------\ndef pod_pgsql_let(start_time: str, pod: px.Pod):\n ''' Calculate LET time-series for PostgreSQL traffic per connection to\n a PostgreSQL database pod.\n\n Calculates latency and throughput for each pod's\n connection to a PostgreSQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for PostgreSQL\n traffic to a PostgreSQL database pod.\n '''\n df = pgsql_let_per_pod(start_time, pod, ['timestamp', 'destination'])\n\n return df[['time_', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput']]\n\n\ndef summary_pgsql_let(start_time: str, pod: px.Pod):\n ''' Calculate LET summary for PostgreSQL traffic per connection to\n a PostgreSQL database pod.\n\n Calculates latency and throughput for each\n connection to a PostgreSQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for PostgreSQL\n traffic to a PostgreSQL database pod.\n '''\n\n df = pgsql_let_per_pod(start_time, pod, ['timestamp', 'source', 'destination',\n 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace'])\n summary_df = summarize_LET(df, ['source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace'])\n summary_df_links = add_source_dest_links(summary_df, start_time)\n\n return summary_df_links[['source', 'destination', 'request_throughput', 'latency',\n 'total_requests']]\n\n\ndef latency_histogram(start_time: str, pod: px.Pod):\n ''' Computes a histogram of PostgreSQL request latency.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc.\n\n Returns: DataFrame of the PostgreSQL latency histogram for svcs that\n match @svc.\n '''\n # The data necessary to compute PostgreSQL LET information is located in the\n # pgsql_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='pgsql_events', start_time=start_time)\n df = format_pgsql_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to PostgreSQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n matching_df.request_latency = px.DurationNanos(px.bin(matching_df.latency,\n latency_bin_size_ns))\n return matching_df.groupby('request_latency').agg(count=('time_', px.count))\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef pgsql_let_per_pod(start_time: str, pod: px.Pod, groups):\n ''' Calculate LET time-series for PostgreSQL traffic per connection to\n a PostgreSQL database pod.\n\n Calculates latency and throughput for each connection to a\n PostgreSQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for PostgreSQL\n traffic to a PostgreSQL database pod.\n '''\n # The data necessary to compute PostgreSQL LET information is located in the\n # pgsql_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='pgsql_events', start_time=start_time)\n df = format_pgsql_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to PostgreSQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n let_df = calc_pgsql_LET(matching_df, groups)\n\n return let_df\n\n\ndef format_events_table(df, latency_col):\n ''' Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency_ms', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on 'pgsql_events' and 'http_events'\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n '''\n df.latency = df[latency_col]\n df.timestamp = px.bin(df.time_, window_ns)\n df = df[df['pod'] != '']\n\n return df\n\n\ndef format_pgsql_table(df):\n ''' Formats pgsql_events tables\n\n Args:\n @df: the input pgsql_events table.\n\n Returns: formatted pgsql_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n return df\n\n\ndef format_LET_aggs(df):\n ''' Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics that\n can be visualized. Latency quantile values need to be extracted from the\n quantiles struct, and then request_throughput is calculated as\n a function of window size.\n\n\n This function represents logic shared by LET calculators for PostgreSQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total' and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n '''\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n\n return df\n\n\ndef calc_pgsql_LET(df, groups):\n ''' Calculates Latency, Error Rate, and Throughput on PostgreSQL events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups.\n\n Args:\n @df: the input pgsql_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n '''\n # All requests for errors and throughput\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef summarize_LET(let_df, groups):\n ''' Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n '''\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n total_requests=('throughput_total', px.sum),\n latency=('latency_p50', px.mean)\n )\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the PostgreSQL request.\n\n PostgreSQL requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the PostgreSQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the PostgreSQL request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to filter by for PostgreSQL request. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_pgsql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P99 Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P99 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Latency Histogram\",\n \"func\": {\n \"name\": \"latency_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.HistogramChart\",\n \"histogram\": {\n \"value\": \"request_latency\",\n \"prebinCount\": \"count\",\n \"maxbins\": 10,\n \"minstep\": 50000000\n },\n \"xAxis\": {\n \"label\": \"Request Latency\"\n },\n \"yAxis\": {\n \"label\": \"# of requests\"\n }\n },\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_pgsql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"PostgreSQL Pod LET metrics","LongDoc":"This live view calculates the latency, error rate, and throughput of a pod's PostgreSQL requests.\n","orgID":"","hidden":false},"px/pid_memory_usage":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n#\n# Get the Virtual memory usage and average memory for all processes in the k8s cluster.\n#\nt1 = px.DataFrame(table='process_stats', start_time='-30s')\n\nt1.timestamp = px.bin(t1.time_, px.seconds(10))\nt1.cmdline = px.upid_to_cmdline(t1.upid)\n\naggop = t1.groupby(['upid', 'timestamp', 'cmdline']).agg(\n vsize=('vsize_bytes', px.mean),\n rss=('rss_bytes', px.mean),\n)\n\n# Format column names.\naggop.pid = px.upid_to_pid(aggop.upid)\naggop.asid = px.upid_to_asid(aggop.upid)\naggop['Process Name'] = aggop.cmdline\n\n# Uncomment and replace number to filter on a specific pid.\n# aggop = aggop[aggop.pid == 4862]\n\naggop['Virtual Memory'] = aggop.vsize\naggop['Average Memory'] = aggop.rss\nkeep_columns = aggop[[\n 'pid',\n 'Process Name',\n 'asid',\n 'timestamp',\n 'Virtual Memory',\n 'Average Memory'\n]]\n\npx.display(keep_columns)\n","vis":"","placement":"","ShortDoc":"Memory Usage of Processes.","LongDoc":"Get the Virtual memory usage and average memory for all processes in the k8s cluster.","orgID":"","hidden":false},"px/pixie_quality_metrics":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\nns_per_ms = 1000 * 1000\n\n\ndef events(t1, latency_col):\n t1.latency_huge = t1[latency_col] \u003e (10 * ns_per_ms)\n t1.negative_latencies = t1[latency_col] \u003c 0\n return t1.groupby(['latency_huge', 'negative_latencies']).agg(count=(latency_col, px.count))\n\n\nt1 = px.DataFrame(table='http_events', start_time='-300s')\npx.display(events(t1, 'latency'), 'http_latencies')\n\nt1 = px.DataFrame(table='mysql_events', start_time='-300s')\npx.display(events(t1, 'latency'), 'mysql_latencies')\n\nt1 = px.DataFrame(table='process_stats', start_time='-30s')\nt1.cmd_line = px.upid_to_cmdline(t1.upid)\nt1 = t1[px.contains(t1.cmd_line, \"java\")]\nt1 = t1.head(1000)\npx.display(t1, 'java_processes')\n\nt1 = px.DataFrame(table='jvm_stats', start_time='-30s')\nt1.cmd_line = px.upid_to_cmdline(t1.upid)\npx.display(t1, 'jvm_stats')\n","vis":"","placement":"","ShortDoc":"Metrics that sample Pixie's collectors","LongDoc":"Metrics that sample Pixie's collector data.","orgID":"","hidden":false},"px/pod":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n'''Pod Overview\n\nOverview of a specific Pod monitored by Pixie with its high level application metrics\n(latency, error-rate \u0026 rps) and resource usage (cpu, writes, reads).\n\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n\n\ndef containers(start_time: str, pod: px.Pod):\n ''' A list of containers in `pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @pod: The name of the pod to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['pod'] == pod]\n df.name = df.ctx['container_name']\n df.id = df.ctx['container_id']\n df = df.groupby(['name', 'id']).agg()\n df.status = px.container_id_to_status(df.id)\n return df\n\n\ndef node(start_time: str, pod: px.Pod):\n ''' A list containing the node the `pod` is running on.\n\n Args:\n @start_time: The timestamp of data to start at.\n @pod: The name of the pod to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['pod'] == pod]\n df.node = df.ctx['node_name']\n df.service = df.ctx['service']\n df.pod_id = df.ctx['pod_id']\n df.pod_name = df.ctx['pod']\n df = df.groupby(['node', 'service', 'pod_id', 'pod_name']).agg()\n df.pod_start_time = px.pod_name_to_start_time(df.pod_name)\n df.status = px.pod_name_to_status(df.pod_name)\n return df.drop('pod_name')\n\n\ndef processes(start_time: str, pod: px.Pod):\n ''' A list of processes in `pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @pod: The name of the pod to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['pod'] == pod]\n df.cmd = df.ctx['cmdline']\n df.pid = df.ctx['pid']\n df = df.groupby(['pid', 'cmd', 'upid']).agg()\n return df\n\n\ndef resource_timeseries(start_time: str, pod: px.Pod):\n ''' Compute the resource usage as a timeseries for `pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @pod: The name of the pod to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['pod'] == pod]\n df.timestamp = px.bin(df.time_, window_ns)\n df.container = df.ctx['container_name']\n\n # First calculate CPU usage by process (UPID) in each k8s_object\n # over all windows.\n df = df.groupby(['upid', 'container', 'timestamp']).agg(\n rss=('rss_bytes', px.mean),\n vsize=('vsize_bytes', px.mean),\n # The fields below are counters, so we take the min and the max to subtract them.\n cpu_utime_ns_max=('cpu_utime_ns', px.max),\n cpu_utime_ns_min=('cpu_utime_ns', px.min),\n cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n read_bytes_max=('read_bytes', px.max),\n read_bytes_min=('read_bytes', px.min),\n write_bytes_max=('write_bytes', px.max),\n write_bytes_min=('write_bytes', px.min),\n rchar_bytes_max=('rchar_bytes', px.max),\n rchar_bytes_min=('rchar_bytes', px.min),\n wchar_bytes_max=('wchar_bytes', px.max),\n wchar_bytes_min=('wchar_bytes', px.min),\n )\n\n\n # Next calculate cpu usage and memory stats per window.\n df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n df.actual_disk_read_throughput = (df.read_bytes_max - df.read_bytes_min) / window_ns\n df.actual_disk_write_throughput = (df.write_bytes_max - df.write_bytes_min) / window_ns\n df.total_disk_read_throughput = (df.rchar_bytes_max - df.rchar_bytes_min) / window_ns\n df.total_disk_write_throughput = (df.wchar_bytes_max - df.wchar_bytes_min) / window_ns\n\n # Then aggregate process individual process metrics.\n df = df.groupby(['timestamp', 'container']).agg(\n cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n cpu_utime_ns=('cpu_utime_ns', px.sum),\n actual_disk_read_throughput=('actual_disk_read_throughput', px.sum),\n actual_disk_write_throughput=('actual_disk_write_throughput', px.sum),\n total_disk_read_throughput=('total_disk_read_throughput', px.sum),\n total_disk_write_throughput=('total_disk_write_throughput', px.sum),\n rss=('rss', px.sum),\n vsize=('vsize', px.sum),\n )\n\n # Finally, calculate total (kernel + user time) percentage used over window.\n df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n df.time_ = df.timestamp\n return df.drop(['cpu_ktime_ns', 'cpu_utime_ns', 'timestamp'])\n\n\ndef network_timeseries(start_time: str, pod: px.Pod):\n ''' Gets the network stats (transmitted/received traffic) for the input node.\n\n Args:\n @start_time Starting time of the data to examine.\n @node: The full name of the node to filter on.\n '''\n df = px.DataFrame(table='network_stats', start_time=start_time)\n df = df[df.ctx['pod'] == pod]\n df.timestamp = px.bin(df.time_, window_ns)\n\n # First calculate network usage by node over all windows.\n # Data is sharded by Pod in network_stats.\n df = df.groupby(['timestamp', 'pod_id']).agg(\n rx_bytes_end=('rx_bytes', px.max),\n rx_bytes_start=('rx_bytes', px.min),\n tx_bytes_end=('tx_bytes', px.max),\n tx_bytes_start=('tx_bytes', px.min),\n tx_errors_end=('tx_errors', px.max),\n tx_errors_start=('tx_errors', px.min),\n rx_errors_end=('rx_errors', px.max),\n rx_errors_start=('rx_errors', px.min),\n tx_drops_end=('tx_drops', px.max),\n tx_drops_start=('tx_drops', px.min),\n rx_drops_end=('rx_drops', px.max),\n rx_drops_start=('rx_drops', px.min),\n )\n\n # Calculate the network statistics rate over the window.\n # We subtract the counter value at the beginning ('_start')\n # from the value at the end ('_end').\n df.rx_bytes_per_ns = (df.rx_bytes_end - df.rx_bytes_start) / window_ns\n df.tx_bytes_per_ns = (df.tx_bytes_end - df.tx_bytes_start) / window_ns\n df.rx_drops_per_ns = (df.rx_drops_end - df.rx_drops_start) / window_ns\n df.tx_drops_per_ns = (df.tx_drops_end - df.tx_drops_start) / window_ns\n df.rx_errors_per_ns = (df.rx_errors_end - df.rx_errors_start) / window_ns\n df.tx_errors_per_ns = (df.tx_errors_end - df.tx_errors_start) / window_ns\n\n # Add up the network values per node.\n df = df.groupby(['timestamp']).agg(\n rx_bytes_per_ns=('rx_bytes_per_ns', px.sum),\n tx_bytes_per_ns=('tx_bytes_per_ns', px.sum),\n rx_drop_per_ns=('rx_drops_per_ns', px.sum),\n tx_drops_per_ns=('tx_drops_per_ns', px.sum),\n rx_errors_per_ns=('rx_errors_per_ns', px.sum),\n tx_errors_per_ns=('tx_errors_per_ns', px.sum),\n )\n df.time_ = df.timestamp\n return df\n\n\ndef inbound_latency_timeseries(start_time: str, pod: px.Pod):\n ''' Compute the latency as a timeseries for requests received by `pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @pod: The name of the pod to filter on.\n\n '''\n df = filtered_http_events(start_time,\n include_health_checks=not filter_health_checks,\n include_ready_checks=not filter_ready_checks,\n include_unresolved_inbound=not filter_unresolved_inbound)\n df.pod = df.ctx['pod']\n df = df[df.pod == pod]\n\n # Filter only to inbound pod traffic (server-side).\n # Don't include traffic initiated by this pod to an external location.\n df = df[df.trace_role == 2]\n\n df.timestamp = px.bin(df.time_, window_ns)\n df = df.groupby(['timestamp']).agg(\n latency_quantiles=('latency', px.quantiles)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.time_ = df.timestamp\n return df[['time_', 'latency_p50', 'latency_p90', 'latency_p99']]\n\n\ndef inbound_request_timeseries_by_container(start_time: str, pod: px.Pod):\n ''' Compute the request statistics as a timeseries for requests received\n by `pod` by container.\n\n Args:\n @start_time: The timestamp of data to start at.\n @pod: The name of the pod to filter on.\n\n '''\n df = filtered_http_events(start_time,\n include_health_checks=not filter_health_checks,\n include_ready_checks=not filter_ready_checks,\n include_unresolved_inbound=not filter_unresolved_inbound)\n df.pod = df.ctx['pod']\n df = df[df.pod == pod]\n\n # Filter only to inbound pod traffic (server-side).\n # Don't include traffic initiated by this pod to an external location.\n df = df[df.trace_role == 2]\n\n df.container = df.ctx['container']\n df.timestamp = px.bin(df.time_, window_ns)\n df = df.groupby(['timestamp', 'container']).agg(\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df.request_throughput = df.throughput_total / window_ns\n df.errors_per_ns = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n df.error_rate = px.Percent(df.error_rate_per_window)\n df.time_ = df.timestamp\n\n return df[['time_', 'container', 'request_throughput', 'errors_per_ns', 'error_rate']]\n\n\ndef inbound_let_summary(start_time: str, pod: px.Pod):\n ''' Gets a summary of requests inbound to `pod`.\n\n Args:\n @start_time Starting time of the data to examine.\n @pod: The pod to filter on.\n '''\n df = filtered_http_events(start_time,\n include_health_checks=not filter_health_checks,\n include_ready_checks=not filter_ready_checks,\n include_unresolved_inbound=not filter_unresolved_inbound)\n df.pod = df.ctx['pod']\n df = df[df.pod == pod]\n\n df = df.groupby(['remote_addr']).agg(\n latency=('latency', px.quantiles),\n total_request_count=('latency', px.count)\n requests_per_window=('time_', px.count),\n error_rate=('failure', px.mean),\n )\n\n df.error_rate = px.Percent(df.error_rate)\n\n # TODO(philkuz): Replace parse_duration with a more general function that accepts all time formats.\n df.start_time = px.now() + px.parse_duration(start_time)\n df.stop_time = px.now()\n df.window = df.stop_time - df.start_time\n df.request_throughput = df.requests_per_window / px.DurationNanos(df.window)\n\n df.requesting_ip = df.remote_addr\n df.requesting_pod_id = px.ip_to_pod_id(df.remote_addr)\n df.requesting_pod = px.pod_id_to_pod_name(df.requesting_pod_id)\n df.requesting_svc = px.pod_id_to_service_name(df.requesting_pod_id)\n\n return df[['requesting_ip', 'requesting_pod', 'requesting_svc', 'latency',\n 'error_rate', 'request_throughput']]\n\n\ndef filtered_http_events(start_time: str, include_health_checks, include_ready_checks, include_unresolved_inbound):\n ''' Returns a dataframe of http events with health/ready requests filtered out and a\n failure field added.\n\n Args:\n @start_time: The timestamp of data to start at.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n df.failure = df.resp_status \u003e= 400\n\n filter_out_conds = ((df.req_path != '/healthz' or include_health_checks) and (\n df.req_path != '/readyz' or include_ready_checks)) and (\n df['remote_addr'] != '-' or include_unresolved_inbound)\n df = df[filter_out_conds]\n\n return df\n\n\ndef stacktraces(start_time: str, pod: str):\n df = px.DataFrame(table='stack_traces.beta', start_time=start_time)\n\n df.namespace = df.ctx['namespace']\n df.pod = df.ctx['pod']\n df.container = df.ctx['container']\n df.cmdline = df.ctx['cmdline']\n\n # Filter on the pod.\n df = df[df.pod == pod]\n\n # Get stack trace totals for the pod.\n # This must be done before any additional filtering to avoid skewing percentages.\n grouping_agg = df.groupby([\"pod\"]).agg(\n count=('count', px.sum)\n )\n\n # Combine flamegraphs from different intervals into one larger framegraph.\n df = df.groupby(['namespace', 'pod', 'container', 'cmdline', 'stack_trace_id']).agg(\n stack_trace=('stack_trace', px.any),\n count=('count', px.sum)\n )\n\n # Compute percentages.\n df = df.merge(\n grouping_agg,\n how='inner',\n left_on=\"pod\",\n right_on=\"pod\",\n suffixes=['', '_x']\n )\n df.percent = 100.0 * df.count / df.count_x\n return df.drop('pod_x')\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The pod name to filter on. Format: \u003cns\u003e/\u003cpod_name\u003e\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"inbound_latency\",\n \"func\": {\n \"name\": \"inbound_latency_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n },\n {\n \"outputName\": \"inbound_requests\",\n \"func\": {\n \"name\": \"inbound_request_timeseries_by_container\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n },\n {\n \"outputName\": \"resource_timeseries\",\n \"func\": {\n \"name\": \"resource_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n },\n {\n \"outputName\": \"network_timeseries\",\n \"func\": {\n \"name\": \"network_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"HTTP Requests\",\n \"globalFuncOutputName\": \"inbound_requests\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"request throughput\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP Errors\",\n \"globalFuncOutputName\": \"inbound_requests\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"errors_per_ns\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"error rate\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP Latency\",\n \"globalFuncOutputName\": \"inbound_latency\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p90\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p99\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"milliseconds\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"CPU Usage\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"cpu_usage\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"CPU usage\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Containers List\",\n \"func\": {\n \"name\": \"containers\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n },\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Process List\",\n \"func\": {\n \"name\": \"processes\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Network Sent and Received\",\n \"globalFuncOutputName\": \"network_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"rx_bytes_per_ns\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"tx_bytes_per_ns\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Network throughput\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Bytes Read\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_read_throughput\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Disk Read Throughput\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 4,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Bytes Written\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_write_throughput\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Disk Write Throughput\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 8,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Resident Set Size\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"rss\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"RSS\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Virtual Memory Size\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"vsize\",\n \"mode\": \"MODE_AREA\",\n \"series\": \"container\",\n \"stackBySeries\": true\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"vsize\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 4,\n \"y\": 9,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Inbound HTTP Traffic to Pod\",\n \"func\": {\n \"name\": \"inbound_let_summary\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 12,\n \"w\": 12,\n \"h\": 3\n }\n },\n {\n \"name\": \"Pod Metadata\",\n \"func\": {\n \"name\": \"node\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 15,\n \"w\": 12,\n \"h\": 2\n }\n },\n {\n \"name\": \"Pod Performance Flamegraph\",\n \"func\": {\n \"name\": \"stacktraces\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.StackTraceFlameGraph\",\n \"stacktraceColumn\": \"stack_trace\",\n \"countColumn\": \"count\",\n \"percentageColumn\": \"percent\",\n \"podColumn\": \"pod\",\n \"containerColumn\": \"container\",\n \"pidColumn\": \"cmdline\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 17,\n \"w\": 12,\n \"h\": 5\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Pod Overview","LongDoc":"Overview of a specific Pod monitored by Pixie with its high level application metrics (latency, error-rate \u0026 rps) and resource usage (cpu, writes, reads).","orgID":"","hidden":false},"px/pod_edge_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Pod Edge LET stats\n\nThis live view calculates LET between a requesting and responding pod.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n# Flag to filter out non_k8s_traffic from the data.\nfilter_non_k8s_traffic = True\n# The bin size in milliseconds to use for the latency histogram.\nlatency_bin_size_ns = 50 * ns_per_ms\n\n\ndef http_requests(start_time: str, requesting_pod: px.Pod, responding_pod: px.Pod):\n \"\"\" Returns HTTP requests from `requesting_pod` to `responding_pod`.\n\n Args:\n @start_time: The start time of the data.\n @requesting_pod: the name of the pod that originates requests.\n @responding_pod: the name of the pod that responds to requests.\n \"\"\"\n\n df = px.DataFrame(table='http_events', start_time=start_time)\n\n # filter to traffic between the requesting and responding pod only.\n df = df[df.ctx['pod'] == responding_pod]\n\n # This join is used as a workaround to filter on `requesting_pod` because\n # px.ip_to_pod_id can only run after a blocking node such as 'agg'.\n remote_ip_table = df.groupby(['remote_addr']).agg()\n remote_ip_table.requesting_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n remote_ip_table = remote_ip_table[remote_ip_table.requesting_pod == requesting_pod]\n\n # remote_ip_table = df[px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr)) == requesting_pod]\n # Inherently filter out IPs that don't belong to `requesting_pod`.\n df = df.merge(remote_ip_table, how='inner', left_on='remote_addr', right_on='remote_addr',\n suffixes=['_x', '']).drop('remote_addr_x')\n\n df.latency = df.latency\n df.timestamp = px.bin(df.time_, window_ns)\n df.failure = df.resp_status \u003e= 400\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n df = df[filter_out_conds]\n return df\n\n\ndef http_requests_formatted(start_time: str, requesting_pod: px.Pod, responding_pod: px.Pod):\n\n df = http_requests(start_time, requesting_pod, responding_pod)\n return df[['timestamp', 'latency', 'req_method', 'req_path', 'req_body',\n 'resp_status', 'resp_body', 'failure']]\n\n\ndef http_code_agg(start_time: str, requesting_pod: px.Pod, responding_pod: px.Pod):\n \"\"\" Computes an aggregate of HTTP status codes for traffic between `requesting_pod`\n and `responding_pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @requesting_pod: The name of the requesting pod.\n @responding_pod: The name of the requesting pod.\n \"\"\"\n df = http_requests(start_time, requesting_pod, responding_pod)\n return df.groupby('resp_status').agg(count=('timestamp', px.count))\n\n\ndef latency_histogram(start_time: str, requesting_pod: px.Pod, responding_pod: px.Pod):\n \"\"\" Computes a histogram of HTTP request latency between `requesting_pod` and\n `responding_pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @requesting_pod: The name of the requesting pod.\n @responding_pod: The name of the requesting pod.\n \"\"\"\n df = http_requests(start_time, requesting_pod, responding_pod)\n df.request_latency = df.latency\n return df.groupby('request_latency').agg(count=('timestamp', px.count))\n\n\ndef pod_edge_let(start_time: str, requesting_pod: px.Pod, responding_pod: px.Pod):\n \"\"\" Computes the latency, error rate, and throughput of traffic between `requesting_pod`\n and `responding_pod`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @requesting_pod: The name of the requesting pod.\n @responding_pod: The name of the requesting pod.\n \"\"\"\n\n df = http_requests(start_time, requesting_pod, responding_pod)\n\n df = df.groupby('timestamp').agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count),\n bytes_recv=('req_body_size', px.sum),\n bytes_sent=('resp_body_size', px.sum)\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n df.req_data_throughput = df.bytes_recv / window_ns\n df.resp_data_throughput = df.bytes_sent / window_ns\n df.error_rate = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"requesting_pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The name of the pod sending the request. Format: ns/svc_name\"\n },\n {\n \"name\": \"responding_pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The name of the pod receiving the request. Format: ns/svc_name\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_edge_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"requesting_pod\",\n \"variable\": \"requesting_pod\"\n },\n {\n \"name\": \"responding_pod\",\n \"variable\": \"responding_pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"HTTP Request Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p90\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p99\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Request Throughput\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Request Error Rate\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Error Rate\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Data Sent/Received\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"resp_data_throughput\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"req_data_throughput\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"http throughtput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Status Code Distribution\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"http_code_agg\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"requesting_pod\",\n \"variable\": \"requesting_pod\"\n },\n {\n \"name\": \"responding_pod\",\n \"variable\": \"responding_pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.BarChart\",\n \"bar\": {\n \"value\": \"count\",\n \"label\": \"resp_status\",\n \"horizontal\": true\n },\n \"xAxis\": {\n \"label\": \"# of requests\"\n },\n \"yAxis\": {\n \"label\": \"HTTP Status Code\"\n }\n }\n },\n {\n \"name\": \"Request Latency Histogram\",\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"latency_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"requesting_pod\",\n \"variable\": \"requesting_pod\"\n },\n {\n \"name\": \"responding_pod\",\n \"variable\": \"responding_pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.HistogramChart\",\n \"histogram\": {\n \"value\": \"request_latency\",\n \"prebinCount\": \"count\",\n \"maxbins\": 10,\n \"minstep\": 50.0\n },\n \"xAxis\": {\n \"label\": \"Request Latency\"\n },\n \"yAxis\": {\n \"label\": \"# of requests\"\n }\n }\n },\n {\n \"name\": \"HTTP Traffic\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"http_requests_formatted\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"requesting_pod\",\n \"variable\": \"requesting_pod\"\n },\n {\n \"name\": \"responding_pod\",\n \"variable\": \"responding_pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Pod to Pod SLAs","LongDoc":"Gets pod latency, error rate and throughput according to another service. Edit the requestor filter to the name of the incoming service you want to filter by. Visualize these in three separate time series charts.\n","orgID":"","hidden":false},"px/pod_lifetime_resource":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n#\n# Returns the Total Resource Usage for each pod.\n#\nt1 = px.DataFrame(table='process_stats', start_time='-60s')\nt1.pod = t1.ctx['pod']\n\nupid_aggop = t1.groupby(['upid', 'pod']).agg(\n vsize=('vsize_bytes', px.mean),\n rss=('rss_bytes', px.mean),\n # The following columns are all counters, so we take the maximum value.\n cpu_utime_ns=('cpu_utime_ns', px.max),\n cpu_ktime_ns=('cpu_ktime_ns', px.max),\n read_bytes=('read_bytes', px.max),\n write_bytes=('write_bytes', px.max),\n rchar_bytes=('rchar_bytes', px.max),\n wchar_bytes=('wchar_bytes', px.max),\n)\n\n# For this aggregate, we sum up the values as we've already calculated the average/usage\n# for the upids already.\npod_aggop = upid_aggop.groupby('pod').agg(\n cpu_utime_ns=('cpu_utime_ns', px.sum),\n cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n vsize=('vsize', px.sum),\n rss=('rss', px.sum),\n read_bytes=('read_bytes', px.sum),\n write_bytes=('write_bytes', px.sum),\n rchar_bytes=('rchar_bytes', px.sum),\n wchar_bytes=('wchar_bytes', px.sum),\n)\n\n# Format everything nicely.\npod_aggop.pod_name = pod_aggop.pod\npod_aggop.status = px.pod_name_to_status(pod_aggop.pod_name)\npod_aggop['Created on'] = px.pod_name_to_start_time(pod_aggop.pod_name)\npod_aggop['CPU User time'] = px.DurationNanos(pod_aggop.cpu_utime_ns)\npod_aggop['CPU System time'] = px.DurationNanos(pod_aggop.cpu_ktime_ns)\npod_aggop['Virtual Memory'] = pod_aggop.vsize\npod_aggop['Average Memory'] = pod_aggop.rss\npod_aggop['Read to IO'] = pod_aggop.read_bytes\npod_aggop['Write to IO'] = pod_aggop.write_bytes\npod_aggop['Characters Read'] = pod_aggop.rchar_bytes\npod_aggop['Characters written'] = pod_aggop.wchar_bytes\n\nkeep_columns = pod_aggop[['pod_name', 'status', 'Created on', 'CPU User time',\n 'CPU System time', 'Virtual Memory',\n 'Average Memory', 'Read to IO', 'Write to IO',\n 'Characters Read', 'Characters written']]\n\npx.display(keep_columns)\n","vis":"","placement":"","ShortDoc":"Total resource usage over pod lifetime.","LongDoc":"Total resource usage of a pod over it's lifetime.","orgID":"","hidden":false},"px/pod_memory_usage":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n###############################################################\n# The following can be edited to k8s object and name to match.\n###############################################################\n# The Kubernetes object to filter on.\n# Options are ['pod', 'service']\nk8s_object = 'pod'\n\n# If you want to filter the object by name, enter the partial\n# or full name here.\nmatch_name = ''\n###############################################################\n\n#\n# Get the virtual and real memory usage for each pod/svc\n# in the cluster.\n#\nt1 = px.DataFrame(table='process_stats', start_time='-1m')\nt1.timestamp = px.bin(t1.time_, px.seconds(10))\nt1[k8s_object] = t1.ctx[k8s_object]\n\nt1 = t1[px.contains(t1[k8s_object], match_name)]\n\nupid_aggop = t1.groupby(['upid', k8s_object, 'timestamp']).agg(\n vsize=('vsize_bytes', px.mean),\n rss=('rss_bytes', px.mean),\n)\n\n# For this aggregate, we sum up the values as we've already calculated the average/usage\n# for the upids already, just need to do it for the entire svc.\naggop = upid_aggop.groupby([k8s_object, 'timestamp']).agg(\n vsize=('vsize', px.sum),\n rss=('rss', px.sum),\n)\n\n# Format column names.\naggop['Virtual Memory'] = aggop.vsize\naggop['Average Memory'] = aggop.rss\nkeep_columns = aggop[[\n k8s_object,\n 'timestamp',\n 'Virtual Memory',\n 'Average Memory'\n]]\n\npx.display(keep_columns)\n","vis":"","placement":"","ShortDoc":"Memory Usage of Processes","LongDoc":"Get the Virtual memory usage and average memory for all processes in the k8s cluster.\n","orgID":"","hidden":false},"px/pods":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n'''Pods Overview\n\nList of Pods monitored by Pixie in a given Namespace with their high level application metrics\n(latency, error-rate \u0026 rps) and resource usage (cpu, writes, reads).\n\n'''\nimport px\n\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n\n\ndef pods(start_time: str, namespace: px.Namespace):\n ''' A list of pods in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The name of the namespace to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['namespace'] == namespace]\n df.pod = df.ctx['pod_name']\n df.container = df.ctx['container_name']\n df.service = df.ctx['service']\n df = df.groupby(['service', 'pod', 'container']).agg()\n df = df.groupby(['service', 'pod']).agg(containers=('container', px.count))\n df.start_time = px.pod_name_to_start_time(df.pod)\n df.status = px.pod_name_to_status(df.pod)\n return df[['pod', 'service', 'start_time', 'containers', 'status']]\n\n\ndef resource_timeseries(start_time: str, namespace: px.Namespace):\n ''' Compute the resource usage as a timeseries for pods in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The name of the namespace to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['namespace'] == namespace]\n df.pod = df.ctx['pod_name']\n df.timestamp = px.bin(df.time_, window_ns)\n\n # First calculate CPU usage by process (UPID) in each k8s_object\n # over all windows.\n df = df.groupby(['upid', 'pod', 'timestamp']).agg(\n rss=('rss_bytes', px.mean),\n vsize=('vsize_bytes', px.mean),\n # The fields below are counters, so we take the min and the max to subtract them.\n cpu_utime_ns_max=('cpu_utime_ns', px.max),\n cpu_utime_ns_min=('cpu_utime_ns', px.min),\n cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n read_bytes_max=('read_bytes', px.max),\n read_bytes_min=('read_bytes', px.min),\n write_bytes_max=('write_bytes', px.max),\n write_bytes_min=('write_bytes', px.min),\n rchar_bytes_max=('rchar_bytes', px.max),\n rchar_bytes_min=('rchar_bytes', px.min),\n wchar_bytes_max=('wchar_bytes', px.max),\n wchar_bytes_min=('wchar_bytes', px.min),\n )\n\n # Next calculate cpu usage and memory stats per window.\n df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n df.actual_disk_read_throughput = (df.read_bytes_max - df.read_bytes_min) / window_ns\n df.actual_disk_write_throughput = (df.write_bytes_max - df.write_bytes_min) / window_ns\n df.total_disk_read_throughput = (df.rchar_bytes_max - df.rchar_bytes_min) / window_ns\n df.total_disk_write_throughput = (df.wchar_bytes_max - df.wchar_bytes_min) / window_ns\n\n # Then aggregate process individual process metrics.\n df = df.groupby(['pod', 'timestamp']).agg(\n cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n cpu_utime_ns=('cpu_utime_ns', px.sum),\n actual_disk_read_throughput=('actual_disk_read_throughput', px.sum),\n actual_disk_write_throughput=('actual_disk_write_throughput', px.sum),\n total_disk_read_throughput=('total_disk_read_throughput', px.sum),\n total_disk_write_throughput=('total_disk_write_throughput', px.sum),\n rss=('rss', px.sum),\n vsize=('vsize', px.sum),\n )\n\n # Finally, calculate total (kernel + user time) percentage used over window.\n df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n df['time_'] = df['timestamp']\n return df.drop(['cpu_ktime_ns', 'cpu_utime_ns', 'timestamp'])\n\n\ndef inbound_let_timeseries(start_time: str, namespace: px.Namespace):\n ''' Compute the let as a timeseries for requests received by pods in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The name of the namespace to filter on.\n\n '''\n df = inbound_let_helper(start_time, namespace)\n\n df = df.groupby(['pod', 'timestamp']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count),\n bytes_total=('resp_body_size', px.sum)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n df.error_rate = px.Percent(df.error_rate_per_window)\n df.bytes_per_ns = df.bytes_total / window_ns\n df.time_ = df.timestamp\n\n return df[['time_', 'pod', 'latency_p50', 'latency_p90', 'latency_p99',\n 'request_throughput', 'error_rate', 'bytes_per_ns']]\n\n\ndef inbound_let_summary(start_time: str, namespace: px.Namespace):\n ''' Gets a summary of request statistics for pods in `namespace`.\n\n Args:\n @start_time Starting time of the data to examine.\n @namespace: The namespace to filter on.\n '''\n df = inbound_let_helper(start_time, namespace)\n\n quantiles_agg = df.groupby(['pod', 'remote_addr']).agg(\n latency=('latency', px.quantiles),\n total_request_count=('latency', px.count)\n )\n quantiles_table = quantiles_agg[['pod', 'remote_addr', 'latency',\n 'total_request_count']]\n\n range_agg = df.groupby(['pod', 'remote_addr', 'timestamp']).agg(\n requests_per_window=('time_', px.count),\n error_rate=('failure', px.mean)\n )\n\n rps_table = range_agg.groupby(['pod', 'remote_addr']).agg(\n requests_per_window=('requests_per_window', px.mean),\n error_rate=('error_rate', px.mean)\n )\n\n joined_table = quantiles_table.merge(rps_table,\n how='inner',\n left_on=['pod', 'remote_addr'],\n right_on=['pod', 'remote_addr'],\n suffixes=['', '_x'])\n\n joined_table.error_rate = px.Percent(joined_table.error_rate)\n joined_table.request_throughput = joined_table.requests_per_window / window_ns\n\n joined_table.responder = df.pod\n joined_table.requesting_ip = df.remote_addr\n joined_table.requesting_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n joined_table.requesting_svc = px.pod_id_to_service_name(px.ip_to_pod_id(df.remote_addr))\n\n return joined_table[['responder', 'requesting_ip', 'requesting_pod', 'requesting_svc',\n 'latency', 'error_rate', 'request_throughput']]\n\n\ndef inbound_let_helper(start_time: str, namespace: px.Namespace):\n ''' Compute the initial part of the let for requests received or sent by pods in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The name of the namespace to filter on.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n\n # Filter only to inbound pod traffic (server-side).\n # Don't include traffic initiated by pods to an external location.\n df = df[df.trace_role == 2]\n\n df = df[df.ctx['namespace'] == namespace]\n df.pod = df.ctx['pod']\n df = df[df.pod != '']\n df.latency = df.latency\n df.timestamp = px.bin(df.time_, window_ns)\n df.failure = df.resp_status \u003e= 400\n\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n df = df[filter_out_conds]\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"inbound_let_timeseries\",\n \"func\": {\n \"name\": \"inbound_let_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n }\n },\n {\n \"outputName\": \"resource_timeseries\",\n \"func\": {\n \"name\": \"resource_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Pods List\",\n \"func\": {\n \"name\": \"pods\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"status\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 8,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP Requests\",\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"request throughput\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP P50 Latency\",\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP P90 Latency\",\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP P99 Latency\",\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P99 Latency\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"HTTP Request Error Rate\",\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"error rate\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"CPU Usage\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"cpu_usage\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"CPU usage\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 4,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Resident Set Size\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"rss\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"RSS Size\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 8,\n \"y\": 6,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Bytes Read from Storage\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_read_throughput\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"bytes read\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Bytes Written to Storage\",\n \"globalFuncOutputName\": \"resource_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"total_disk_write_throughput\",\n \"series\": \"pod\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"bytes written\"\n },\n \"xAxis\": null\n },\n \"position\": {\n \"x\": 4,\n \"y\": 9,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Inbound HTTP Traffic to Pods\",\n \"func\": {\n \"name\": \"inbound_let_summary\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"position\": {\n \"x\": 0,\n \"y\": 12,\n \"w\": 12,\n \"h\": 3\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Pods Overview","LongDoc":"List of Pods monitored by Pixie in a given Namespace with their high level application metrics (latency, error-rate \u0026 rps) and resource usage (cpu, writes, reads).","orgID":"","hidden":false},"px/redis_data":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Redis Data Tracer\n\nShows the most recent Redis messages in the cluster.\n'''\nimport px\n\n\ndef redis_data(start_time: str, source_filter: str, destination_filter: str, num_head: int):\n\n df = px.DataFrame(table='redis_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Add additional filters below:\n\n # Restrict number of results.\n df = df.head(num_head)\n\n df = add_source_dest_links(df, start_time)\n df = df[['time_', 'source', 'destination', 'remote_port', 'req_cmd',\n 'req_args', 'resp', 'latency']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Redis request.\n\n Redis requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Redis request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Redis request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"redis_data\",\n \"func\": {\n \"name\": \"redis_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"redis_data\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Redis RPC messages","LongDoc":"Shows a sample of Redis messages in the cluster.\n","orgID":"","hidden":false},"px/redis_flow_graph":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Redis Overview Map\n\nShows a graph of Redis requests in the cluster, with some latency information.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef redis_flow_graph(start_time: str, ns: px.Namespace, source_filter: str,\n destination_filter: str):\n\n df = px.DataFrame('redis_events', start_time=start_time)\n df = add_source_dest_columns(df)\n\n # Filter on namespace as specified by the user.\n df = df[df.namespace == ns]\n\n # Filter out entities as specified by the user.\n df = df[px.contains(df.source, source_filter)]\n df = df[px.contains(df.destination, destination_filter)]\n\n # Create 10 ns bin for time_ column\n df.timestamp = px.bin(df.time_, window_ns)\n\n df = df.groupby(['timestamp', 'source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace']).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count),\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n\n df = df.groupby(['source', 'destination', 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n return df\n\n\ndef redis_summary_with_links(start_time: str, ns: px.Namespace, source_filter: str,\n destination_filter: str):\n\n df = redis_flow_graph(start_time, ns, source_filter, destination_filter)\n df = add_source_dest_links(df, start_time)\n df = df[['source', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput', 'throughput_total']]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Redis request.\n\n Redis requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Redis request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Redis request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The namespace to filter on.\"\n },\n {\n \"name\": \"source_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"destination_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'destination' column.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"redis_flow\",\n \"func\": {\n \"name\": \"redis_flow_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n },\n {\n \"outputName\": \"redis_summary_with_links\",\n \"func\": {\n \"name\": \"redis_summary_with_links\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"ns\",\n \"variable\": \"namespace\"\n },\n {\n \"name\": \"source_filter\",\n \"variable\": \"source_filter\"\n },\n {\n \"name\": \"destination_filter\",\n \"variable\": \"destination_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Redis Flow Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"redis_flow\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"source\",\n \"toColumn\": \"destination\"\n },\n \"edgeWeightColumn\": \"request_throughput\",\n \"edgeColorColumn\": \"latency_p90\",\n \"edgeThresholds\": {\n \"mediumThreshold\": 100000000,\n \"highThreshold\": 500000000\n },\n \"edgeHoverInfo\": [\n \"latency_p50\",\n \"latency_p90\",\n \"latency_p99\",\n \"request_throughput\",\n \"throughput_total\"\n ],\n \"edgeLength\": 500\n }\n },\n {\n \"name\": \"Table\",\n \"position\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"redis_summary_with_links\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Redis Flow Graph","LongDoc":"Graph of Redis messages in the cluster, with latency stats.\n","orgID":"","hidden":false},"px/redis_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Redis Pod LET metrics\n\nThis live view calculates the latency, error rate, and throughput\nfor each connection to a Redis database pod.\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(50 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization specification, vis.json.\n# ----------------------------------------------------------------\ndef pod_redis_let(start_time: str, pod: px.Pod):\n ''' Calculate LET time-series for Redis traffic per connection to\n a Redis database pod.\n\n Calculates latency and throughput for each pod's\n connection to a Redis database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor Redis LET.\n\n Returns: Returns the DataFrame containing LET time-series for Redis\n traffic to a Redis database pod.\n '''\n df = redis_let_per_pod(start_time, pod, ['timestamp', 'destination'])\n\n return df[['time_', 'destination', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput']]\n\n\ndef summary_redis_let(start_time: str, pod: px.Pod):\n ''' Calculate LET summary for redis traffic per connection to\n a Redis database pod.\n\n Calculates latency and throughput for each\n connection to a Redis database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor redis LET.\n\n Returns: Returns the DataFrame containing LET time-series for Redis\n traffic to a Redis database pod.\n '''\n\n df = redis_let_per_pod(start_time, pod, ['timestamp', 'source', 'destination',\n 'is_source_pod_type', 'is_dest_pod_type',\n 'namespace'])\n summary_df = summarize_LET(df, ['source', 'destination', 'is_source_pod_type',\n 'is_dest_pod_type', 'namespace'])\n summary_df_links = add_source_dest_links(summary_df, start_time)\n\n return summary_df_links[['source', 'destination', 'request_throughput', 'latency',\n 'total_requests']]\n\n\ndef latency_histogram(start_time: str, pod: px.Pod):\n ''' Computes a histogram of Redis request latency.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc.\n\n Returns: DataFrame of the Redis latency histogram for svcs that\n match @svc.\n '''\n # The data necessary to compute redis LET information is located in the\n # redis_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='redis_events', start_time=start_time)\n df = format_redis_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to redis connections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n matching_df.request_latency = px.DurationNanos(px.bin(matching_df.latency,\n latency_bin_size_ns))\n return matching_df.groupby('request_latency').agg(count=('time_', px.count))\n\n\ndef cmd_timeseries(start_time: str, pod: px.Pod):\n ''' Calculate Redis cmd timeseries for redis traffic per connection to\n a Redis database pod.\n\n @start_time The timestamp of data to start at.\n @pod: The partial/full-name of the pod to monitor.\n\n Returns: Returns the DataFrame containing Redis cmd time-series for Redis\n traffic to a Redis database pod.\n '''\n df = px.DataFrame(table='redis_events', start_time=start_time)\n df = format_redis_table(df)\n df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n df.time_ = px.bin(df.time_, window_ns)\n df = df.groupby(['time_', 'req_cmd']).agg(\n throughput_total=('req_cmd', px.count),\n )\n return df\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef redis_let_per_pod(start_time: str, pod: px.Pod, groups):\n ''' Calculate LET time-series for redis traffic per connection to\n a Redis database pod.\n\n Calculates latency and throughput for each connection to a\n Redis database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor redis LET.\n\n Returns: Returns the DataFrame containing LET time-series for Redis\n traffic to a Redis database pod.\n '''\n # The data necessary to compute redis LET information is located in the\n # redis_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='redis_events', start_time=start_time)\n df = format_redis_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to redis connections\n # over the time window ('timestamp') after filtering for matching svcs.\n matching_df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n let_df = calc_redis_LET(matching_df, groups)\n\n return let_df\n\n\ndef format_events_table(df, latency_col):\n ''' Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency_ms', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on 'redis_events' and 'http_events'\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n '''\n df.latency = df[latency_col]\n df.timestamp = px.bin(df.time_, window_ns)\n df = df[df['pod'] != '']\n\n return df\n\n\ndef format_redis_table(df):\n ''' Formats redis_events tables\n\n Args:\n @df: the input redis_events table.\n\n Returns: formatted redis_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n return df\n\n\ndef format_LET_aggs(df):\n ''' Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics that\n can be visualized. Latency quantile values need to be extracted from the\n quantiles struct, and then request_throughput is calculated as\n a function of window size.\n\n\n This function represents logic shared by LET calculators for redis and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total' and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n '''\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n\n return df\n\n\ndef calc_redis_LET(df, groups):\n ''' Calculates Latency, Error Rate, and Throughput on redis events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups.\n\n Args:\n @df: the input redis_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n '''\n # All requests for errors and throughput\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef summarize_LET(let_df, groups):\n ''' Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n '''\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n total_requests=('throughput_total', px.sum),\n latency=('latency_p50', px.mean)\n )\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the Redis request.\n\n Redis requests are traced server-side (trace_role==2), unless the server is\n outside of the cluster in which case the request is traced client-side (trace_role==1).\n\n When trace_role==2, the Redis request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the Redis request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to filter by for redis request. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_redis_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P99 Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P99 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"destination\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput per Cmd\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"cmd_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"throughput_total\",\n \"series\": \"req_cmd\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Cmd throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Latency Histogram\",\n \"func\": {\n \"name\": \"latency_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.HistogramChart\",\n \"histogram\": {\n \"value\": \"request_latency\",\n \"prebinCount\": \"count\",\n \"maxbins\": 10,\n \"minstep\": 50000000\n },\n \"xAxis\": {\n \"label\": \"Request Latency\"\n },\n \"yAxis\": {\n \"label\": \"# of requests\"\n }\n },\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_redis_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Redis Pod LET metrics","LongDoc":"This live view calculates the latency, error rate, and throughput of a pod's Redis requests.\n","orgID":"","hidden":false},"px/schemas":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\n# get the Pixide datasource table descriptions\ndef table_desc():\n return px.GetTables()\n\n\n# get the Pixie datasource table schema\ndef table_schema():\n return px.GetSchemas()\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [],\n \"widgets\": [\n {\n \"name\": \"Pixie Datasource Table Descriptions\",\n \"func\": {\n \"name\": \"table_desc\",\n \"args\": []\n },\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Pixie Datasource Table Schema\",\n \"func\": {\n \"name\": \"table_schema\",\n \"args\": []\n },\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 4\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Get all the table schemas available in the system","LongDoc":"Get all the table schemas available in the system\n","orgID":"","hidden":false},"px/service":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Service Overview\n\n This script gets an overview of an individual service, summarizing its request statistics.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n\n\ndef pods_for_service(start_time: str, service: px.Service):\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[px.has_service_name(df.ctx['service'], service)]\n df.pod = df.ctx['pod_name']\n df = df.groupby('pod').agg()\n df.pod_create_time = px.pod_name_to_start_time(df.pod)\n df.pod_status = px.pod_name_to_status(df.pod)\n return df\n\n\ndef inbound_let_timeseries(start_time: str, service: px.Service):\n ''' Compute the let as a timeseries for requests received by `service`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @service: The name of the service to filter on.\n\n '''\n df = let_helper(start_time)\n df = df[px.has_service_name(df.service, service)]\n\n df = df.groupby(['timestamp']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count),\n bytes_total=('resp_body_size', px.sum)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n df.errors_per_ns = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n df.error_rate = px.Percent(df.error_rate_per_window)\n df.bytes_per_ns = df.bytes_total / window_ns\n df.time_ = df.timestamp\n\n return df[['time_', 'latency_p50', 'latency_p90', 'latency_p99',\n 'request_throughput', 'errors_per_ns', 'error_rate', 'bytes_per_ns']]\n\n\ndef inbound_let_summary(start_time: str, service: px.Service):\n ''' Gets a summary of requests inbound to `service`.\n\n Args:\n @start_time Starting time of the data to examine.\n @service: The service to filter on.\n '''\n df = let_summary_helper(start_time)\n df = df[px.has_service_name(df.responder, service)]\n return df.drop('responder')\n\n\ndef let_summary_helper(start_time: str):\n ''' Gets a summary of request statistics. This is a helper function, filtering by\n service is done elsewhere.\n\n Args:\n @start_time Starting time of the data to examine.\n '''\n df = let_helper(start_time)\n\n df = df.groupby(['service', 'remote_addr']).agg(\n latency=('latency', px.quantiles),\n total_request_count=('latency', px.count)\n error_rate=('failure', px.mean),\n )\n\n df.error_rate = px.Percent(df.error_rate)\n df.responder = df.service\n df.requesting_ip = df.remote_addr\n df.requesting_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.requesting_svc = px.pod_id_to_service_name(px.ip_to_pod_id(df.remote_addr))\n\n return df[['responder', 'requesting_ip', 'requesting_pod', 'requesting_svc',\n 'latency', 'error_rate']]\n\n\ndef service_slow_requests(start_time: str, service: px.Service):\n df = let_helper(start_time)\n df = df[px.has_service_name(df.service, service)]\n quantiles = df.groupby('service').agg(\n latency_quantiles=('latency', px.quantiles)\n )\n quantiles.service_p99 = px.pluck_float64(quantiles.latency_quantiles, 'p99')\n quantiles = quantiles.drop('latency_quantiles')\n requests = df.merge(quantiles, left_on='service', right_on='service', how='inner',\n suffixes=['', '_x'])\n requests = requests[requests.latency \u003e= px.floor(requests.service_p99)]\n\n return requests[['time_', 'pod', 'latency', 'req_method',\n 'req_path', 'req_body', 'resp_status',\n 'remote_addr', 'remote_port',\n 'resp_body']].head(100)\n\n\ndef let_helper(start_time: str):\n ''' Compute the initial part of the let for requests.\n Filtering to inbound/outbound traffic by service is done by the calling function.\n\n Args:\n @start_time: The timestamp of data to start at.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.trace_role == 2]\n df.service = df.ctx['service']\n df.pod = df.ctx['pod']\n df.latency = df.latency\n\n df.timestamp = px.bin(df.time_, window_ns)\n\n df.failure = df.resp_status \u003e= 400\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n\n df = df[filter_out_conds]\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"service\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The name of the service to get stats for. Format: ns/svc_name\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"inbound_let_timeseries\",\n \"func\": {\n \"name\": \"inbound_let_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"HTTP Requests\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Errors\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"errors_per_ns\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"error rate\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p90\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p99\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Response Data Throughput\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"bytes_per_ns\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Response data throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Pod List\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 8,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"pods_for_service\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\",\n \"gutterColumn\": \"pod_status\"\n }\n },\n {\n \"name\": \"Inbound HTTP Traffic to Service\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"inbound_let_summary\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Sample of Slow Inbound Requests\",\n \"position\": {\n \"x\": 0,\n \"y\": 12,\n \"w\": 12,\n \"h\": 9\n },\n \"func\": {\n \"name\": \"service_slow_requests\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Service Overview","LongDoc":"This script gets an overview of an individual service, summarizing its request statistics.\n","orgID":"","hidden":false},"px/service_edge_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Service Edge LET stats\n\nThis live view calculates LET between two Services.\n\nNotes:\n* Setting requesting_svc or responding_svc does not exclusively match\n at the moment.\n IE: if you have two svcs 'a' and 'a-db', setting `svc_name = 'a'`\n will still match `a-db`\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\n# K8s object is the abstraction to group on.\n# Options are ['pod', 'service'].\nk8s_object = 'service'\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n# Flag to filter out non_k8s_traffic from the data.\nfilter_non_k8s_traffic = True\n# The name of the incoming traffic column in the edge graphs.\nsrc_col = 'requestor'\n# The name of the outgoing traffic column in the edge graphs.\ndest_col = 'responder'\n# Column naem used to split data into separate time series.\n# k8s_object column is renamed to this and is used in\n# visualization spec.\nsplit_series_name = 'k8s'\n###############################################################\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization speciciation, vis.json.\n# ----------------------------------------------------------------\ndef svc_edge_let(start_time: str, requesting_svc: px.Service,\n responding_svc: px.Service):\n \"\"\" Returns the LET time-series for each edge that matches the filter.\n\n For each edge in the svc graph that matches the filter, we calculate the latency,\n error rate, and throughput as a time-series.\n\n A matching edge will start at a svc that matches @requesting_svc\n and end at a svc that matches @responding_svc\n\n Args:\n @start_time The timestamp of data to start at.\n @requesting_svc: the partial/full-name of the svc(s)\n that originate requests. (start of the edge).\n @responding_svc: the partial/full-name of the svc(s)\n to receive requests. (end of the edge).\n\n Returns: DataFrame of the LET stats for edges in the svc graph.\n \"\"\"\n df = make_http_table(start_time)\n\n # Calculate LET of svc(s) (k8s_object) over the time window ('timestamp')\n # after filtering for matching svcs.\n matching_df = df[px.contains(df[k8s_object], responding_svc)]\n let_df = let_per_edge(matching_df)\n # Filter the DataFrame to only those svcs that match the requesting svc.\n let_df = let_df[px.contains(let_df[src_col], requesting_svc)]\n let_df[dest_col] = let_df[k8s_object]\n let_df = let_df['time_', src_col, split_series_name, dest_col, 'latency_p50',\n 'latency_p90', 'latency_p99', 'error_rate',\n 'request_throughput', 'bytes_throughput']\n return let_df\n\n\ndef summary_edge_let(start_time: str, requesting_svc: px.Service,\n responding_svc: px.Service):\n \"\"\" Returns the summary of LET statistics for matching edges.\n\n\n For each edge in the svc graph that matches the filter, we calculate the latency,\n error rate, and throughput as an avg over the examined\n data window.\n\n A matching edge will start at a svc that matches @requesting_svc\n and end at a svc that matches @responding_svc\n\n Args:\n @start_time The timestamp of data to start at.\n @requesting_svc: the partial/full-name of the svc(s)\n that originate requests. (start of the edge).\n @responding_svc: the partial/full-name of the svc(s)\n to receive requests. (end of the edge).\n\n Returns: DataFrame of the LET stats for edges in the svc graph.\n \"\"\"\n df = svc_edge_let(start_time, requesting_svc, responding_svc)\n return summarize_LET(df, [src_col, dest_col])\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef let_per_edge(df: px.DataFrame):\n \"\"\" Calculates the LET per edge of the svc graph.\n\n Args:\n @start_time The timestamp of data to start at.\n\n Returns: DataFrame of HTTP events with formatted columns.\n \"\"\"\n # Calculate LET for each svc edge in the svc graph over each time window.\n # Each edge starts at a requester ('remote_addr') and ends at a\n # responder (k8s_object).\n edge_let_df = calc_http_LET(df, ['remote_addr', k8s_object, 'timestamp'])\n # Convert 'remote_addr' IP into a svc name.\n edge_let_df = ip_to_svc_name(edge_let_df, 'remote_addr', src_col)\n # Rename k8s_object to dest_col.\n edge_let_df[dest_col] = edge_let_df[k8s_object]\n # Format the LET DataFrame columns.\n edge_let_df[split_series_name] = edge_let_df[k8s_object]\n return edge_let_df\n\n\ndef make_http_table(start_time: str):\n \"\"\" Makes the HTTP table given the passed in start.\n\n The data necessary to compute HTTP level svc information is located in the\n http_events table. We filter and aggregate data from this table to compute the\n required metrics.\n\n Args:\n @start_time The timestamp of data to start at.\n\n Returns: DataFrame of HTTP events with formatted columns.\n \"\"\"\n # The data necessary to compute HTTP level svc information is located in the\n # http_events table. We filter and aggregate data from this table to compute the\n # required metrics.\n df = px.DataFrame(table='http_events', start_time=start_time)\n df = format_http_table(df, filter_health_checks,\n filter_ready_checks, filter_unresolved_inbound)\n return df\n\n\ndef format_events_table(df, latency_col):\n \"\"\" Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on \"mysql_events\" and \"http_events\"\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n \"\"\"\n df.latency = df[latency_col]\n\n df.timestamp = px.bin(df.time_, window_ns)\n df[k8s_object] = df.ctx[k8s_object]\n df = df[df[k8s_object] != '']\n return df\n\n\ndef format_http_table(df, filter_health_checks, filter_ready_checks,\n filter_unresolved_inbound):\n \"\"\" Formats HTTP events tables\n\n Runs events table universal formatting, adds a response_size,\n creates a failure field marking which requests receive an error\n status code, and optionally filters out system monitoring requests\n and partial data points.\n\n Args:\n @df: the input http_events table.\n @filter_health_checks: flag to filter health checks.\n @filter_ready_checks: flag to filter health checks.\n @filter_unresolved_inbound: flag to filter unresolved inbound\n requests.\n\n Returns: formatted HTTP events DataFrame.\n \"\"\"\n df = format_events_table(df, 'latency')\n df.failure = df.resp_status \u003e= 400\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n\n df = df[filter_out_conds]\n return df\n\n\ndef format_LET_aggs(df):\n \"\"\" Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics that\n can be visualized. Latency quantile values need to be extracted from the\n quantiles struct, and then error_rate, request_throughput, and bytes_throughput\n are calculated as a function of window size.\n\n\n This function represents logic shared by LET calculators for MySQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total', 'error_rate_per_window', and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n \"\"\"\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n df.bytes_throughput = df.bytes_total / window_ns\n df.error_rate = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n\n return df\n\n\ndef calc_http_LET(df, groups):\n \"\"\" Calculates Latency, Error Rate, and Throughput on HTTP events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups. Throughput is represented by two values: request_throughput, and bytes_throughput.\n\n Args:\n @df: the input http_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n \"\"\"\n # Aggregate values over the window.\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count),\n bytes_total=('resp_body_size', px.sum)\n )\n df.bytes_total = px.Bytes(df.bytes_total)\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef ip_to_pod_name(df, ip_col, pod_col_name):\n \"\"\" Map IP to pod name.\n\n Maps IP values stored in @ip_col into pod names to store into\n @pod_col_name.\n\n Args:\n @df: the input dataframe.\n @ip_col: the IP column to map from.\n @pod_col_name: the column name to assign the new pod values.\n\n Returns: DataFrame with the pod_col added.\n \"\"\"\n pod_id = 'pod_id'\n df[pod_id] = px.ip_to_pod_id(df[ip_col])\n df[pod_col_name] = px.pod_id_to_pod_name(df[pod_id])\n return df.drop(pod_id)\n\n\ndef ip_to_svc_name(df, ip_col, svc_col_name):\n \"\"\" Map IP to svc name.\n\n Maps IP values stored in @ip_col into svc names to store into\n @svc_col_name.\n\n Args:\n @df: the input dataframe.\n @ip_col: the IP column to map from.\n @svc_col_name: the column name to assign the new svc values.\n\n Returns: DataFrame with the svc_col added.\n \"\"\"\n pod_id = 'pod_id'\n df[pod_id] = px.ip_to_pod_id(df[ip_col])\n df[svc_col_name] = px.pod_id_to_service_name(df[pod_id])\n return df.drop(pod_id)\n\n\ndef summarize_LET(let_df, groups):\n \"\"\" Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n \"\"\"\n\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n bytes_throughput=('bytes_throughput', px.mean),\n error_rate=('error_rate', px.mean),\n )\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"requesting_svc\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The full/partial name of the service to for request side. Format: ns/svc_name\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"responding_svc\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The full/partial name of the service to for response side. Format: ns/svc_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"svc_edge_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"requesting_svc\",\n \"variable\": \"requesting_svc\"\n },\n {\n \"name\": \"responding_svc\",\n \"variable\": \"responding_svc\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"p50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"p90 Latency\",\n \"position\": {\n \"x\": 6,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Error Rate\",\n \"position\": {\n \"x\": 6,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Error Rate\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_edge_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"requesting_svc\",\n \"variable\": \"requesting_svc\"\n },\n {\n \"name\": \"responding_svc\",\n \"variable\": \"responding_svc\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Basic Service to Service SLAs","LongDoc":"Gets service latency, error rate and throughput according to another service. Edit the requestor filter to the name of the incoming service you want to filter by. Visualize these in three separate time series charts.\n","orgID":"","hidden":false},"px/service_resource_usage":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n# Window size to use on time_ column for bucketing.\nns_per_s = 1000 * 1000 * 1000\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n\ndef services_for_cluster(start_time: str):\n pods = process_stats_by_service(start_time)\n service_let = inbound_service_http_traffic_summary(start_time)\n joined = pods.merge(service_let, how='left', left_on='service', right_on='service',\n suffixes=['', '_x'])\n joined.http_request_throughput = joined.http_throughput_total / px.DurationNanos(joined.time_window)\n return joined[['service', 'pod_count', 'avg_pod_cpu', 'avg_pod_rss', 'http_request_throughput', 'http_latency']]\n\n\ndef process_stats_by_service(start_time: str):\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df.pod = df.ctx['pod']\n df.service = df.ctx['service']\n df = df[df.service != '']\n df.timestamp = px.bin(df.time_, window_ns)\n df = df.groupby(['service', 'pod', 'upid']).agg(\n time_min=('time_', px.min),\n time_max=('time_', px.max),\n avg_upid_rss=('rss_bytes', px.mean),\n avg_upid_vsz=('vsize_bytes', px.mean),\n # The fields below are counters, so we take the min and the max to subtract them.\n cpu_utime_ns_max=('cpu_utime_ns', px.max),\n cpu_utime_ns_min=('cpu_utime_ns', px.min),\n cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n cpu_ktime_ns_min=('cpu_ktime_ns', px.min)\n )\n df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n # Sum by UPID.\n df = df.groupby(['service', 'pod']).agg(\n time_min=('time_min', px.min),\n time_max=('time_max', px.max),\n cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n cpu_utime_ns=('cpu_utime_ns', px.sum),\n avg_rss=('avg_upid_rss', px.sum),\n avg_vsz=('avg_upid_vsz', px.sum)\n )\n # Calculate total (kernel + user time) percentage used over window.\n df.time_window = df.time_max - df.time_min\n df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / df.time_window)\n # Take averages across service.\n df = df.groupby('service').agg(avg_pod_cpu=('cpu_usage', px.mean),\n avg_pod_rss=('avg_rss', px.mean),\n avg_pod_vsz=('avg_vsz', px.mean),\n pod_count=('pod', px.count),\n time_window=('time_window', px.max))\n return df\n\n\ndef inbound_service_http_traffic_summary(start_time: str):\n df = px.DataFrame(table='http_events', start_time=start_time)\n df.service = df.ctx['service']\n df.pod = df.ctx['pod']\n\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.service != '']\n df = df[df.trace_role == 2]\n df = df.groupby('service').agg(\n http_throughput_total=('latency', px.count),\n http_latency=('latency', px.quantiles)\n )\n return df\n\n\ndf = services_for_cluster('-10m')\npx.display(df)\n","vis":"","placement":"","ShortDoc":"Resource Usage of Services.","LongDoc":"Get the average pod CPU, pod memory, HTTP throughput and latency by service.","orgID":"","hidden":false},"px/service_stats":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Service stats LET and local svc map\n\nThis live view calculates the latency, error rate, and throughput\nof a Service and also summarizes the incoming and outgoing traffic\nfor a svc.\n\nNotes:\n* Setting svc is not exclusive matching at the moment. IE\n if you have two svcs 'a' and 'a-db', setting `svc = 'a'`\n will still match `a-db`\n'''\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\n# K8s object is the abstraction to group on.\n# Options are ['pod', 'service'].\nk8s_object = 'service'\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n# Flag to filter out non_k8s_traffic from the data.\nfilter_non_k8s_traffic = True\n# The name of the incoming traffic column in the edge graphs.\nsrc_col = 'requestor'\n# The name of the outgoing traffic column in the edge graphs.\ndest_col = 'responder'\n# Column naem used to split data into separate time series.\n# k8s_object column is renamed to this and is used in\n# visualization spec.\nsplit_series_name = 'k8s'\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(50 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\ndef svc_let(start_time: str, svc: px.Service):\n \"\"\" Calculates LET filtered on the svc\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc to\n calculate LET.\n\n Returns: DataFrame of the LET stats for svcs that\n match @svc.\n \"\"\"\n df = make_http_table(start_time)\n # Calculate LET of svc(s) (k8s_object) over the time window ('timestamp')\n # after filtering for matching svcs.\n matching_df = df[px.contains(df[k8s_object], svc)]\n let_df = calc_http_LET(matching_df, [k8s_object, 'timestamp'])\n\n # Format and organize resulting columns.\n let_df[split_series_name] = let_df[k8s_object]\n let_df = let_df[['time_', split_series_name, 'latency_p50',\n 'latency_p90', 'latency_p99', 'error_rate',\n 'request_throughput', 'bytes_throughput']]\n return let_df\n\n\ndef http_code_histogram(start_time: str, svc: px.Service):\n \"\"\" Computes a histogram of HTTP status codes\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc.\n\n Returns: DataFrame of the HTTP status code stats for svcs that\n match @svc.\n \"\"\"\n df = make_http_table(start_time)\n matching_df = df[px.contains(df[k8s_object], svc)]\n return matching_df.groupby(['resp_status']).agg(count=('latency', px.count))\n\n\ndef latency_histogram(start_time: str, svc: px.Service):\n \"\"\" Computes a histogram of HTTP request latency.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc.\n\n Returns: DataFrame of the HTTP latency histogram for svcs that\n match @svc.\n \"\"\"\n df = make_http_table(start_time)\n matching_df = df[px.contains(df[k8s_object], svc)]\n matching_df.request_latency = px.DurationNanos(px.bin(matching_df.latency,\n latency_bin_size_ns))\n return matching_df.groupby('request_latency').agg(count=('resp_status', px.count))\n\n\ndef outgoing_edges(start_time: str, svc: px.Service):\n \"\"\" Determines the svcs that this svc makes\n requests to.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc to\n get outgoing_edges.\n\n Returns: DataFrame of the svcs this svc talks to\n and the LET summary of that communication.\n \"\"\"\n df = let_per_edge(start_time)\n # Filter for traffic that starts at this svc.\n outgoing_let_df = df[px.contains(\n df[src_col], svc)]\n\n # Group outgoing traffic by (src, dest) to get all outgoing edges.\n return summarize_LET(outgoing_let_df, [src_col, dest_col])\n\n\ndef incoming_edges(start_time: str, svc: px.Service):\n \"\"\" Determines the svcs that make requests to this svc.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc to\n get incoming_edges.\n\n Returns: DataFrame of the svcs that talk to this svc\n and the LET summary of that communication.\n \"\"\"\n df = let_per_edge(start_time)\n\n # Filter traffic that ends at this svc.\n incoming_let_df = df[px.contains(\n df[dest_col], svc)]\n\n # Group incoming traffic by (src, dest) to get all incoming edges.\n return summarize_LET(incoming_let_df, [src_col, dest_col])\n\n\ndef svc_graph(start_time: str, svc: px.Service):\n \"\"\" Determines the svcs that make requests to this svc.\n Like incoming_edges, but filters out empty strings in the src/dest.\n\n Args:\n @start_time The timestamp of data to start at.\n @svc: the partial/full-name of the svc to\n get svc_graph in.\n\n Returns: DataFrame of the svcs that talk to this svc\n and the LET summary of that communication.\n \"\"\"\n\n df = incoming_edges(start_time, svc)\n return df[df[src_col] != '' and df[dest_col] != '']\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef make_http_table(start_time: str):\n \"\"\" Makes the HTTP table given the passed in start.\n\n The data necessary to compute HTTP level svc information is located in the\n http_events table. We filter and aggregate data from this table to compute the\n required metrics.\n\n Args:\n @start_time The timestamp of data to start at.\n\n Returns: DataFrame of HTTP events with formatted columns.\n \"\"\"\n df = px.DataFrame(table='http_events', start_time=start_time)\n df = format_http_table(df, filter_health_checks,\n filter_ready_checks, filter_unresolved_inbound)\n return df\n\n\ndef let_per_edge(start_time: str):\n \"\"\" Calculates the LET per edge of the svc graph.\n\n Args:\n @start_time The timestamp of data to start at.\n\n Returns: DataFrame of HTTP events with formatted columns.\n \"\"\"\n\n df = make_http_table(start_time)\n # Calculate LET for each svc edge in the svc graph over each time window.\n # Each edge starts at a requester ('remote_addr') and ends at a\n # responder (k8s_object).\n edge_let_df = calc_http_LET(df, ['remote_addr', k8s_object, 'timestamp'])\n # Convert 'remote_addr' IP into a svc name.\n edge_let_df = ip_to_svc_name(edge_let_df, 'remote_addr', src_col)\n # Rename k8s_object to dest_col.\n edge_let_df[dest_col] = edge_let_df[k8s_object]\n return edge_let_df\n\n\ndef format_events_table(df, latency_col):\n \"\"\" Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on \"mysql_events\" and \"http_events\"\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n \"\"\"\n df.latency = df[latency_col]\n\n df.timestamp = px.bin(df.time_, window_ns)\n df[k8s_object] = df.ctx[k8s_object]\n df = df[df[k8s_object] != '']\n return df\n\n\ndef format_http_table(df, filter_health_checks, filter_ready_checks,\n filter_unresolved_inbound):\n \"\"\" Formats HTTP events tables\n\n Runs events table universal formatting, adds a response_size,\n creates a failure field marking which requests receive an error\n status code, and optionally filters out system monitoring requests\n and partial data points.\n\n Args:\n @df: the input http_events table.\n @filter_health_checks: flag to filter health checks.\n @filter_ready_checks: flag to filter health checks.\n @filter_unresolved_inbound: flag to filter unresolved inbound\n requests.\n\n Returns: formatted HTTP events DataFrame.\n \"\"\"\n df = format_events_table(df, 'latency')\n df.failure = df.resp_status \u003e= 400\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n\n df = df[filter_out_conds]\n return df\n\n\ndef format_LET_aggs(df):\n \"\"\" Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics that\n can be visualized. Latency quantile values need to be extracted from the\n quantiles struct, and then error_rate, request_throughput, and bytes_throughput\n are calculated as a function of window size.\n\n\n This function represents logic shared by LET calculators for MySQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total', 'error_rate_per_window', and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n \"\"\"\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n df.bytes_throughput = df.bytes_total / window_ns\n df.error_rate = df.error_rate_per_window * df.request_throughput / px.DurationNanos(1)\n\n return df\n\n\ndef calc_http_LET(df, groups):\n \"\"\" Calculates Latency, Error Rate, and Throughput on HTTP events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups. Throughput is represented by two values: request_throughput, and bytes_throughput.\n\n Args:\n @df: the input http_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n \"\"\"\n # Aggregate values over the window.\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count),\n bytes_total=('resp_body_size', px.sum)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef ip_to_svc_name(df, ip_col, svc_col_name):\n \"\"\" Map IP to svc name.\n\n Maps IP values stored in @ip_col into svc names to store into\n @svc_col_name.\n\n Args:\n @df: the input dataframe.\n @ip_col: the IP column to map from.\n @svc_col_name: the column name to assign the new svc values.\n\n Returns: DataFrame with the svc_col added.\n \"\"\"\n pod_id = 'pod_id'\n df[pod_id] = px.ip_to_pod_id(df[ip_col])\n df[svc_col_name] = px.pod_id_to_service_name(df[pod_id])\n return df.drop(pod_id)\n\n\ndef summarize_LET(let_df, groups):\n \"\"\" Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n \"\"\"\n\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n bytes_throughput=('bytes_throughput', px.mean),\n error_rate=('error_rate', px.mean),\n )\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"svc\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The full/partial name of the service to get stats for. Format: ns/svc_name\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"svc_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"p50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"p90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Error Rate\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"series\": \"k8s\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Error Rate\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Status Code Distribution\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"http_code_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.BarChart\",\n \"bar\": {\n \"value\": \"count\",\n \"label\": \"resp_status\",\n \"horizontal\": true\n },\n \"xAxis\": {\n \"label\": \"# of requests\"\n },\n \"yAxis\": {\n \"label\": \"HTTP Status Code\"\n }\n }\n },\n {\n \"name\": \"Request Latency Histogram\",\n \"func\": {\n \"name\": \"latency_histogram\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.HistogramChart\",\n \"histogram\": {\n \"value\": \"request_latency\",\n \"prebinCount\": \"count\",\n \"maxbins\": 10,\n \"minstep\": 50000000\n },\n \"xAxis\": {\n \"label\": \"Request Latency\"\n },\n \"yAxis\": {\n \"label\": \"# of requests\"\n }\n },\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n }\n },\n {\n \"name\": \"Incoming Traffic\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 6,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"incoming_edges\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Outgoing Traffic\",\n \"position\": {\n \"x\": 6,\n \"y\": 6,\n \"w\": 6,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"outgoing_edges\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Service Graph\",\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"svc_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"svc\",\n \"variable\": \"svc\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"requestor\",\n \"toColumn\": \"responder\"\n }\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Basic service SLAs.","LongDoc":"Gets service latency, error rate and throughput. Visualize them in three separate time series charts.\n","orgID":"","hidden":false},"px/services":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Services Overview\n\n This script gets an overview all of the services in `namespace`.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n# Whether or not to include traffic from IPs that don't resolve to a known pod/service.\ninclude_ips = True\n\n\ndef services(start_time: str, namespace: px.Namespace):\n ''' Get a list of the services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df.service = df.ctx['service']\n df = df[df.ctx['namespace'] == namespace and df.service != '']\n df.pod = df.ctx['pod']\n df = df.groupby(['service', 'pod']).agg()\n return df.groupby('service').agg(pod_count=('pod', px.count))\n\n\ndef inbound_service_let(start_time: str, namespace: px.Namespace):\n ''' Compute the let as a timeseries for requests received or by services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n\n # Calculate LET for each svc edge in the svc graph over each time window.\n # Each edge starts at a requester ('remote_addr') and ends at a\n # responder service.\n\n df = inbound_service_let_helper(start_time, namespace)\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.trace_role == 2]\n df = df.groupby(['timestamp', 'service']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate=('failure', px.mean),\n throughput_total=('latency', px.count),\n inbound_bytes_total=('req_body_size', px.sum),\n outbound_bytes_total=('resp_body_size', px.sum)\n )\n\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n\n df.request_throughput = df.throughput_total / window_ns\n df.inbound_throughput = df.inbound_bytes_total / window_ns\n df.outbound_throughput = df.outbound_bytes_total / window_ns\n df.error_rate = px.Percent(df.error_rate)\n df.time_ = df.timestamp\n\n return df[['time_', 'service', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput', 'error_rate',\n 'inbound_throughput', 'outbound_throughput']]\n\n\ndef inbound_let_summary(start_time: str, namespace: px.Namespace):\n ''' Compute a summary of inbound traffic by requesting service, for requests\n on services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = inbound_service_let_helper(start_time, namespace)\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.trace_role == 2]\n df = df[df.service != '']\n df.responder = df.service\n df.requesting_ip = df.remote_addr\n df.requesting_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.requesting_svc = px.pod_id_to_service_name(px.ip_to_pod_id(df.remote_addr))\n\n per_ns_df = df.groupby(['timestamp', 'requesting_ip', 'requesting_pod', 'requesting_svc',\n 'responder']).agg(\n throughput_total=('latency', px.count),\n inbound_bytes_total=('req_body_size', px.sum),\n outbound_bytes_total=('resp_body_size', px.sum)\n )\n\n per_ns_df.request_throughput = per_ns_df.throughput_total / window_ns\n per_ns_df.inbound_throughput = per_ns_df.inbound_bytes_total / window_ns\n per_ns_df.outbound_throughput = per_ns_df.outbound_bytes_total / window_ns\n\n per_ns_df = per_ns_df.groupby(['requesting_ip', 'requesting_pod', 'requesting_svc',\n 'responder']).agg(\n request_throughput=('request_throughput', px.mean),\n inbound_throughput=('inbound_throughput', px.mean),\n outbound_throughput=('outbound_throughput', px.mean)\n )\n\n quantiles_df = df.groupby(['requesting_ip', 'requesting_pod', 'requesting_svc',\n 'responder']).agg(\n latency=('latency', px.quantiles)\n error_rate=('failure', px.mean),\n )\n\n quantiles_df.error_rate = px.Percent(quantiles_df.error_rate)\n\n joined = per_ns_df.merge(quantiles_df, left_on=['requesting_ip', 'responder'],\n right_on=['requesting_ip', 'responder'], how='inner',\n suffixes=['', '_x'])\n return joined[['requesting_ip', 'requesting_pod', 'requesting_svc', 'responder', 'latency',\n 'request_throughput', 'error_rate', 'inbound_throughput', 'outbound_throughput']]\n\n\ndef inbound_let_service_graph(start_time: str, namespace: px.Namespace):\n ''' Compute a summary of traffic by requesting service, for requests on services in `namespace`.\n Similar to `inbound_let_summary` but also breaks down by pod in addition to service.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = inbound_service_let_helper(start_time, namespace)\n df = df.groupby(['timestamp', 'service', 'remote_addr', 'pod', 'trace_role']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate=('failure', px.mean),\n throughput_total=('latency', px.count),\n inbound_bytes_total=('req_body_size', px.sum),\n outbound_bytes_total=('resp_body_size', px.sum)\n )\n\n # Get the traced and remote pod/service/IP information.\n df.traced_pod = df.pod\n df.traced_service = df.service\n df.traced_ip = px.pod_name_to_pod_ip(df.pod)\n\n localhost_ip_regexp = r'127\\.0\\.0\\.[0-9]+'\n df.is_remote_addr_localhost = px.regex_match(localhost_ip_regexp, df.remote_addr)\n df.remote_pod = px.select(df.is_remote_addr_localhost,\n df.pod,\n px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr)))\n df.remote_service = px.select(df.is_remote_addr_localhost,\n df.traced_service,\n px.service_id_to_service_name(px.ip_to_service_id(df.remote_addr)))\n\n df.remote_ip = df.remote_addr\n # If external IPs are excluded in the service graph, then we also exclude any\n # traffic where we don't know the remote pod or remote service name.\n df = df[include_ips or (df.remote_pod != '' or df.remote_service != '')]\n\n # Associate it with Client/Server roles, based on the trace role.\n df.is_server_side_tracing = df.trace_role == 2\n df.responder_pod = px.select(df.is_server_side_tracing, df.traced_pod, df.remote_pod)\n df.requestor_pod = px.select(df.is_server_side_tracing, df.remote_pod, df.traced_pod)\n df.responder_service = px.select(df.is_server_side_tracing, df.traced_service, df.remote_service)\n df.requestor_service = px.select(df.is_server_side_tracing, df.remote_service, df.traced_service)\n df.responder_ip = px.select(df.is_server_side_tracing, df.traced_ip, df.remote_ip)\n df.requestor_ip = px.select(df.is_server_side_tracing, df.remote_ip, df.traced_ip)\n\n # Compute statistics about each edge of the service graph.\n df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n df.inbound_throughput = df.inbound_bytes_total / window_ns\n df.outbound_throughput = df.outbound_bytes_total / window_ns\n df.error_rate = px.Percent(df.error_rate)\n\n return df.groupby(['responder_pod', 'requestor_pod', 'responder_service',\n 'requestor_service', 'responder_ip', 'requestor_ip']).agg(\n latency_p50=('latency_p50', px.mean),\n latency_p90=('latency_p90', px.mean),\n latency_p99=('latency_p99', px.mean),\n request_throughput=('request_throughput', px.mean),\n error_rate=('error_rate', px.mean),\n inbound_throughput=('inbound_throughput', px.mean),\n outbound_throughput=('outbound_throughput', px.mean),\n throughput_total=('throughput_total', px.sum)\n )\n\n\ndef inbound_service_let_helper(start_time: str, namespace: px.Namespace):\n ''' Compute the let as a timeseries for requests received or by services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n df.service = df.ctx['service']\n df.pod = df.ctx['pod_name']\n df = df[df.ctx['namespace'] == namespace]\n df = df[df.pod != '']\n df.latency = df.latency\n df.timestamp = px.bin(df.time_, window_ns)\n\n df.failure = df.resp_status \u003e= 400\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n\n df = df[filter_out_conds]\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The name of the namespace to get stats for.\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"inbound_service_let\",\n \"func\": {\n \"name\": \"inbound_service_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Services\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"services\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"HTTP Request Throughput\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_service_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"service\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"requests per second\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Request Error Rate\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_service_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"service\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"error rate (%)\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_service_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"service\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Mean latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP P90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_service_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"service\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Mean latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP P99 Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"inbound_service_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"service\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Mean latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Service Map\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"inbound_let_service_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.RequestGraph\",\n \"requestorPodColumn\": \"requestor_pod\",\n \"responderPodColumn\": \"responder_pod\",\n \"requestorServiceColumn\": \"requestor_service\",\n \"responderServiceColumn\": \"responder_service\",\n \"requestorIPColumn\": \"requestor_ip\",\n \"responderIPColumn\": \"responder_ip\",\n \"p50Column\": \"latency_p50\",\n \"p90Column\": \"latency_p90\",\n \"p99Column\": \"latency_p99\",\n \"errorRateColumn\": \"error_rate\",\n \"requestsPerSecondColumn\": \"request_throughput\",\n \"inboundBytesPerSecondColumn\": \"inbound_throughput\",\n \"outboundBytesPerSecondColumn\": \"outbound_throughput\",\n \"totalRequestCountColumn\": \"throughput_total\"\n }\n },\n {\n \"name\": \"Inbound HTTP Traffic to Services\",\n \"position\": {\n \"x\": 0,\n \"y\": 9,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"inbound_let_summary\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Services Overview","LongDoc":"This script gets an overview of the services in a namespace, summarizing their request statistics.\n","orgID":"","hidden":false},"px/slow_http_requests":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = True\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = True\n\nns_per_ms = 1000 * 1000\n\n\ndef namespace_slow_requests(start_time: str, namespace: px.Namespace):\n df = px.DataFrame(table='http_events', start_time=start_time)\n df.service = df.ctx['service']\n df = df[df.ctx['namespace'] == namespace and df.service != '']\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n filter_out_conds = (((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)) and (\n df.latency \u003e 100 * ns_per_ms)\n df = df[filter_out_conds]\n\n quantiles = df.groupby('service').agg(\n latency_quantiles=('latency', px.quantiles)\n )\n quantiles.service_p99 = px.pluck_float64(quantiles.latency_quantiles, 'p99')\n quantiles = quantiles.drop('latency_quantiles')\n requests = df.merge(quantiles, left_on='service', right_on='service', how='inner',\n suffixes=['', '_x'])\n requests = requests[requests.latency \u003e= px.floor(requests.service_p99)]\n return requests[['time_', 'source', 'destination', 'remote_port', 'latency', 'req_method',\n 'req_path', 'resp_status', 'resp_body']].head(1000)\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns.\n\n Connections are traced server-side (trace_role==2), unless the server is outside\n of the cluster in which case the connection is traced client-side (trace_role==1).\n\n With trace_role==2: source is the `remote_addr` col, destination is the `pod` col.\n With trace_role==1: source is the `pod` col, destination is the `remote_addr` col.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/ip script showing all network connections\n to/from that IP address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.source,\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/ip', {\n 'start_time': start_time,\n 'ip': df.destination,\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The name of the namespace to get stats for.\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Sample of Slow Requests by Service\",\n \"position\": {\n \"x\": 0,\n \"y\": 12,\n \"w\": 12,\n \"h\": 9\n },\n \"func\": {\n \"name\": \"namespace_slow_requests\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Sample of Slow Requests","LongDoc":"This view shows a sample of slow requests by service.\n","orgID":"","hidden":false},"px/sql_queries":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n# Copyright (c) Pixie Labs, Inc.\n# Licensed under the Apache License, Version 2.0 (the \"License\")\n\n'''\nNormalized SQL Queries\n\nThis live view calculates the latency, error rate, and throughput\nof each distinct normalized SQL Query.\n'''\n\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(50 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization specification, vis.json.\n# ----------------------------------------------------------------\ndef pod_sql_let(start_time: str, pod: px.Pod):\n ''' Calculate LET time-series for SQL traffic per connection to\n a PostgreSQL or MySQL database pod.\n\n Calculates latency and throughput for each pod's\n connection to a PostgreSQL or MySQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for\n traffic to a PostgreSQL or MySQL database pod grouped by\n normalized query.\n '''\n df = merged_let_per_pod(start_time, pod)\n\n return df[['time_', 'normed_query', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput']]\n\n\ndef summary_sql_let(start_time: str, pod: px.Pod):\n ''' Calculate LET summary for PostgreSQL/MySQL traffic per connection to\n a PostgreSQL/MySQL database pod.\n\n Calculates latency and throughput for each\n connection to a PostgreSQL/MySQL database pod.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing LET time-series for\n PostgreSQL/MySQL traffic to a PostgreSQL/MySQL database pod.\n '''\n\n df = merged_let_per_pod(start_time, pod)\n df = summarize_LET(df, ['normed_query'])\n df = add_query_links(df, start_time, pod)\n\n return df[['normed_query', 'request_throughput',\n 'latency', 'total_requests']]\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef merged_let_per_pod(start_time: str, pod: px.Pod):\n pgsql_df = normalized_pgsql_per_pod(start_time, pod)\n mysql_df = normalized_mysql_per_pod(start_time, pod)\n\n common_cols = ['timestamp', 'normed_query', 'latency']\n pgsql_df = pgsql_df[common_cols]\n mysql_df = mysql_df[common_cols]\n\n # we don't apriori know whether a pod is a postgres or mysql pod,\n # so we merge the two event tables together and then calculate our LET timeseries.\n df = pgsql_df.append(mysql_df)\n\n # TODO(james, PP-2582): once we have list addition we can make groups\n # an argument to this function, similar to how it works in\n # mysql_stats/pgsql_stats. Without list addition we can't do this\n # because we need to determine column_cols as:\n # groups + ['latency']\n groups = ['timestamp', 'normed_query']\n df = calc_LET(df, groups)\n return df\n\n\ndef normalized_pgsql_per_pod(start_time: str, pod: px.Pod):\n ''' Normalized PostgreSQL traffic per connection to a\n PostgreSQL database pod.\n\n Formats the table and normalizes queries in preparation for\n merge with MySQL table.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing normalized PostgresSQL queries\n and unaggregated LET statistics.\n '''\n # The data necessary to compute PostgreSQL LET information is located in the\n # pgsql_events table. We filter and aggregate data from this table to\n # compute the required metrics.\n df = px.DataFrame(table='pgsql_events', start_time=start_time)\n df = format_pgsql_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to PostgreSQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n df = filter_pgsql_events(df)\n df = normalize_pgsql(df)\n return df\n\n\ndef normalized_mysql_per_pod(start_time: str, pod: px.Pod):\n ''' Normalized MySQL traffic per connection to a MySQL database pod.\n\n Formats the table and normalizes queries in preparation for merge with\n PostgresSQL table.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor MySQL LET.\n\n Returns: Returns the DataFrame containing normalized MySQL queries\n and unaggregated LET statistics.\n '''\n # The data necessary to compute MySQL LET information is located in the\n # mysql_events table. We filter and aggregate data from this\n # table to compute the required metrics.\n df = px.DataFrame(table='mysql_events', start_time=start_time)\n df = format_mysql_table(df, False)\n\n # Calculate LET of pod(s) (k8s_object) connection to PostgreSQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n df = filter_mysql_events(df)\n df = normalize_mysql(df)\n return df\n\n\ndef format_events_table(df, latency_col):\n ''' Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency_ms', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on 'pgsql_events' and 'http_events'\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n '''\n df.latency = df[latency_col]\n df.timestamp = px.bin(df.time_, window_ns)\n df = df[df['pod'] != '']\n\n return df\n\n\ndef format_pgsql_table(df):\n ''' Formats pgsql_events tables\n\n Args:\n @df: the input pgsql_events table.\n\n Returns: formatted pgsql_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n return df\n\n\ndef format_mysql_table(df, filter_responses_with_none_code):\n ''' Formats mysql_events tables\n\n Runs events table universal formatting, creates a failure field\n marking which requests receive an error status code, and\n optionally filters out responses with \"None\" response\n error code if @filter_responses_with_none_code is true.\n\n Args:\n @df: the input mysql_events table.\n @filter_responses_with_none_code: flag to filter out MySQL\n responses where no data is returned.\n Returns: formatted mysql_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n df.failure = df['resp_status'] == 3\n\n # None response code returns on close() of MySQL connection.\n none_response_code = 1\n df = df[df.resp_status\n != none_response_code or not filter_responses_with_none_code]\n return df\n\n\ndef filter_pgsql_events(df):\n ''' Filters out irrelevant PostgresSQL events.\n\n For now we only consider QUERY and EXECUTE events.\n Args:\n @df: the input pgsql_events table.\n Returns: filtered table.\n '''\n return df[df.req_cmd == 'Execute' or df.req_cmd == 'Query']\n\n\ndef filter_mysql_events(df):\n ''' Filters out irrelevant MySQL events.\n\n For now we only consider COM_QUERY and COM_STMT_EXECUTE events.\n Args:\n @df: the input mysql_events table.\n Returns: filtered table.\n '''\n # COM_QUERY has command code 3 and COM_STMT_EXECUTE has command code 23.\n return df[df.req_cmd == 3 or df.req_cmd == 23]\n\n\ndef format_LET_aggs(df):\n ''' Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics\n that can be visualized. Latency quantile values need to be extracted from\n the quantiles struct, and then request_throughput is calculated as a function\n of window size.\n\n\n This function represents logic shared by LET calculators for PostgreSQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total' and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n '''\n df.latency_p50 = px.DurationNanos(px.floor(\n px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(\n px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(\n px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n\n return df\n\n\ndef calc_LET(df, groups):\n ''' Calculates Latency, Error Rate, and Throughput on\n PostgreSQL/MySQL events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups.\n\n Args:\n @df: the input pgsql_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n '''\n # All requests for errors and throughput\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef normalize_pgsql(df):\n # Normalize queries\n df.normed_query_struct = px.normalize_pgsql(df.req, df.req_cmd)\n df.normed_query = px.pluck(df.normed_query_struct, 'query')\n df.params = px.pluck(df.normed_query_struct, 'params')\n\n df = df[df.normed_query != \"\"]\n df = df.drop(['normed_query_struct'])\n return df\n\n\ndef normalize_mysql(df):\n df.normed_query_struct = px.normalize_mysql(df.req_body, df.req_cmd)\n df.normed_query = px.pluck(df.normed_query_struct, 'query')\n df.params = px.pluck(df.normed_query_struct, 'params')\n\n df = df[df.normed_query != \"\"]\n df = df.drop(['normed_query_struct'])\n return df\n\n\ndef summarize_LET(let_df, groups):\n ''' Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n '''\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n total_requests=('throughput_total', px.sum),\n latency=('latency_p50', px.mean)\n )\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the PostgreSQL request.\n\n PostgreSQL and MySQL requests are traced server-side (trace_role==2),\n unless the server is outside of the cluster in which case the request\n is traced client-side (trace_role==1).\n\n When trace_role==2, the request source is the remote_addr column\n and destination is the pod column. When trace_role==1, the request\n source is the pod column and the destination is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_query_links(df, start_time: str, pod: px.Pod):\n ''' Modifies the normed_query column to be a deeplink to\n the px/sql_query script.\n '''\n\n df.normed_query = px.script_reference(df.normed_query, 'px/sql_query', {\n 'start': start_time,\n 'pod': pod,\n 'query': df.normed_query,\n })\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to filter by for SQL request. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_sql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"normed_query\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P90 Latency\",\n \"position\": {\n \"x\": 6,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"normed_query\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P99 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"series\": \"normed_query\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P99 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 6,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"normed_query\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_sql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Normalized SQL Queries","LongDoc":"This live view calculates the latency, error rate, and throughput of each distinct normalized SQL Query. Only supports Postgres or MySQL.\n","orgID":"","hidden":false},"px/sql_query":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n# Copyright (c) Pixie Labs, Inc.\n# Licensed under the Apache License, Version 2.0 (the \"License\")\n\n'''\nNormalized SQL Queries\n\nThis live view calculates the latency, error rate, and throughput\nof each distinct normalized SQL Query.\n'''\n\nimport px\n\n# ----------------------------------------------------------------\n# Visualization Variables - No need to edit for basic configuration.\n# ----------------------------------------------------------------\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# The bin size to use for the latency histogram.\nlatency_bin_size_ns = px.DurationNanos(50 * ns_per_ms)\n# ----------------------------------------------------------------\n\n\n# ----------------------------------------------------------------\n# Visualization functions:\n#\n# These functions are formatted and ready for use in\n# the visualization specification, vis.json.\n# ----------------------------------------------------------------\ndef pod_sql_let(start_time: str, pod: px.Pod, normed_query: str):\n ''' Calculate LET time-series for PostgreSQL or MySQL traffic\n for a given normalized SQL query.\n\n Calculates latency and throughput for each parameter set to the query.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor LET.\n @normed_query: the normalized SQL query to monitor.\n\n Returns: Returns the DataFrame containing LET time-series for\n traffic to a PostgreSQL or MySQL database pod grouped by query params.\n '''\n df = merged_let_per_pod(start_time, pod)\n df = df[df.normed_query == normed_query]\n\n return df[['time_', 'normed_query', 'params', 'latency_p50',\n 'latency_p90', 'latency_p99', 'request_throughput']]\n\n\ndef summary_sql_let(start_time: str, pod: px.Pod, normed_query: str):\n ''' Calculate LET summary for PostgreSQL or MySQL traffic\n for a given normalized SQL query.\n\n Calculates latency and throughput for each parameter set to the query.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor LET.\n @normed_query: the normalized SQL query to monitor.\n\n Returns: Returns the DataFrame containing LET summary statistics\n for the query grouped by parameter set.\n '''\n\n df = merged_let_per_pod(start_time, pod)\n df = df[df.normed_query == normed_query]\n df = summarize_LET(df, ['normed_query', 'params'])\n\n return df[['params', 'normed_query',\n 'request_throughput', 'latency', 'total_requests']]\n\n\n# ----------------------------------------------------------------\n# Utility functions:\n#\n# These are shared functions. We plan to support imports in v0.3,\n# which will allow these functions to be shared across multiple\n# scripts.\n# ----------------------------------------------------------------\ndef merged_let_per_pod(start_time: str, pod: px.Pod):\n pgsql_df = normalized_pgsql_per_pod(start_time, pod)\n mysql_df = normalized_mysql_per_pod(start_time, pod)\n\n common_cols = ['timestamp', 'normed_query', 'params', 'latency']\n pgsql_df = pgsql_df[common_cols]\n mysql_df = mysql_df[common_cols]\n\n # we don't apriori know whether a pod is a postgres or mysql pod,\n # so we merge the two event tables together\n # and then calculate our LET timeseries.\n df = pgsql_df.append(mysql_df)\n\n # TODO(james, PP-2582): once we have list addition we can make\n # groups an argument to this function, similar to how it works\n # in mysql_stats/pgsql_stats.\n # Without list addition we can't do this cause we need\n # to determine column_cols as groups + ['latency']\n groups = ['timestamp', 'normed_query', 'params']\n df = calc_LET(df, groups)\n return df\n\n\ndef normalized_pgsql_per_pod(start_time: str, pod: px.Pod):\n ''' Normalized traffic per connection to a PostgreSQL database pod.\n\n Formats the table and normalizes queries.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor PostgreSQL LET.\n\n Returns: Returns the DataFrame containing normalized queries\n and unaggregated LET statistics.\n '''\n # The data necessary to compute PostgreSQL LET information is located in the\n # pgsql_events table. We filter and aggregate data\n # from this table to compute the required metrics.\n df = px.DataFrame(table='pgsql_events', start_time=start_time)\n df = format_pgsql_table(df)\n\n # Calculate LET of pod(s) (k8s_object) connection to PostgreSQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n df = filter_pgsql_events(df)\n df = normalize_pgsql(df)\n return df\n\n\ndef normalized_mysql_per_pod(start_time: str, pod: px.Pod):\n ''' Normalized MySQL traffic per connection to a MySQL database pod.\n\n Formats the table and normalizes queries.\n\n @start_time The timestamp of data to start at.\n @pod: the partial/full-name of the pod to monitor MySQL LET.\n\n Returns: Returns the DataFrame containing normalized MySQL queries\n and unaggregated LET statistics.\n '''\n # The data necessary to compute MySQL LET information is located in the\n # mysql_events table. We filter and aggregate data\n # from this table to compute the required metrics.\n df = px.DataFrame(table='mysql_events', start_time=start_time)\n df = format_mysql_table(df, False)\n\n # Calculate LET of pod(s) (k8s_object) connection to PostgreSQL connections\n # over the time window ('timestamp') after filtering for matching svcs.\n df = df[px.contains(df.source, pod) or px.contains(df.destination, pod)]\n\n df = filter_mysql_events(df)\n df = normalize_mysql(df)\n return df\n\n\ndef format_events_table(df, latency_col):\n ''' Format data and add semantic columns in event tables\n\n Unifies latency column to 'latency_ms', adds a binned\n timestamp field to aggregate on, and adds the svc\n (k8s_object) as a semantic column.\n\n Works on 'pgsql_events' and 'http_events'\n\n Args:\n @df: the input events table\n @latency_col: the name of the latency column in @df.\n\n Returns: formatted events DataFrame\n '''\n df.latency = df[latency_col]\n df.timestamp = px.bin(df.time_, window_ns)\n df = df[df['pod'] != '']\n\n return df\n\n\ndef format_pgsql_table(df):\n ''' Formats pgsql_events tables\n\n Args:\n @df: the input pgsql_events table.\n\n Returns: formatted pgsql_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n return df\n\n\ndef format_mysql_table(df, filter_responses_with_none_code):\n ''' Formats mysql_events tables\n\n Runs events table universal formatting, creates a failure field\n marking which requests receive an error status code, and\n optionally filters out responses with \"None\" response\n error code if @filter_responses_with_none_code is true.\n\n Args:\n @df: the input mysql_events table.\n @filter_responses_with_none_code: flag to filter out MySQL\n responses where no data is returned.\n Returns: formatted mysql_events DataFrame.\n '''\n # Add source, destination columns.\n df = add_source_dest_columns(df)\n\n df = format_events_table(df, 'latency')\n df.failure = df['resp_status'] == 3\n\n # None response code returns on close() of MySQL connection.\n none_response_code = 1\n df = df[df.resp_status\n != none_response_code or not filter_responses_with_none_code]\n return df\n\n\ndef filter_pgsql_events(df):\n ''' Filters out irrelevant PostgresSQL events.\n\n For now we only consider QUERY and EXECUTE events.\n Args:\n @df: the input pgsql_events table.\n Returns: filtered table.\n '''\n return df[df.req_cmd == 'Execute' or df.req_cmd == 'Query']\n\n\ndef filter_mysql_events(df):\n ''' Filters out irrelevant MySQL events.\n\n For now we only consider COM_QUERY and COM_STMT_EXECUTE events.\n Args:\n @df: the input mysql_events table.\n Returns: filtered table.\n '''\n # COM_QUERY has command code 3 and COM_STMT_EXECUTE has command code 23.\n return df[df.req_cmd == 3 or df.req_cmd == 23]\n\n\ndef format_LET_aggs(df):\n ''' Converts the result of LET windowed aggregates into expected metrics.\n\n Converts the result of aggregates on windows into well-formatted metrics\n that can be visualized. Latency quantile values need to be extracted\n from the quantiles struct, and then request_throughput is calculated\n as a function of window size.\n\n\n This function represents logic shared by LET calculators for PostgreSQL and\n HTTP events.\n\n Args:\n @df: the input events table grouped into windows with aggregated\n columns 'throughput_total' and 'request_throughput'\n\n Returns: DataFrame with formatted LET metrics.\n '''\n df.latency_p50 = px.DurationNanos(px.floor(\n px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(px.floor(\n px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(px.floor(\n px.pluck_float64(df.latency_quantiles, 'p99')))\n df['time_'] = df['timestamp']\n df.request_throughput = df.throughput_total / window_ns\n\n return df\n\n\ndef calc_LET(df, groups):\n ''' Calculates Latency, Error Rate, and Throughput on PostgreSQL events.\n\n Calculates latency, error rate, and throughput aggregated over\n @groups.\n\n Args:\n @df: the input pgsql_events table.\n @groups: the list of columns to group on. 'timestamp' must be a a group\n or this will fail.\n\n Returns: The LET DataFrame.\n '''\n # All requests for errors and throughput\n df = df.groupby(groups).agg(\n latency_quantiles=('latency', px.quantiles),\n throughput_total=('latency', px.count)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df = format_LET_aggs(df)\n return df\n\n\ndef normalize_pgsql(df):\n # Normalize queries\n df.normed_query_struct = px.normalize_pgsql(df.req, df.req_cmd)\n df.normed_query = px.pluck(df.normed_query_struct, 'query')\n df.params = px.pluck(df.normed_query_struct, 'params')\n\n df = df[df.normed_query != \"\"]\n df = df.drop(['normed_query_struct'])\n return df\n\n\ndef normalize_mysql(df):\n df.normed_query_struct = px.normalize_mysql(df.req_body, df.req_cmd)\n df.normed_query = px.pluck(df.normed_query_struct, 'query')\n df.params = px.pluck(df.normed_query_struct, 'params')\n\n df = df[df.normed_query != \"\"]\n df = df.drop(['normed_query_struct'])\n return df\n\n\ndef summarize_LET(let_df, groups):\n ''' Aggregate LET values across all windows.\n\n Args:\n @let_df: the DataFrame with LET values.\n @groups: the columns to group over.\n\n Returns: The summary DF.\n '''\n df = let_df.groupby(groups).agg(\n request_throughput=('request_throughput', px.mean),\n total_requests=('throughput_total', px.sum),\n latency=('latency_p50', px.mean)\n )\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns for the PostgreSQL request.\n\n PostgreSQL requests are traced server-side (trace_role==2),\n unless the server is outside of the cluster in which case the request\n is traced client-side (trace_role==1).\n\n When trace_role==2, the PostgreSQL request source is the remote_addr column\n and destination is the pod column. When trace_role==1,\n the PostgreSQL request source is the pod column and the destination\n is the remote_addr column.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"pod\",\n \"type\": \"PX_POD\",\n \"description\": \"The full/partial name of the pod to filter by for SQL request. Format: ns/pod_name\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"query\",\n \"type\": \"PX_STRING\",\n \"description\": \"The normalzied SQL query. Eg. SELECT * FROM foo WHERE bar=$1\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"LET\",\n \"func\": {\n \"name\": \"pod_sql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n },\n {\n \"name\": \"normed_query\",\n \"variable\": \"query\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"series\": \"params\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P50 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P90 Latency\",\n \"position\": {\n \"x\": 6,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"series\": \"params\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P90 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"P99 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"series\": \"params\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"P99 Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Request Throughput\",\n \"position\": {\n \"x\": 6,\n \"y\": 3,\n \"w\": 6,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"LET\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"series\": \"params\",\n \"stackBySeries\": false,\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Summary\",\n \"position\": {\n \"x\": 0,\n \"y\": 6,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"summary_sql_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"pod\",\n \"variable\": \"pod\"\n },\n {\n \"name\": \"normed_query\",\n \"variable\": \"query\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"SQL Query Params","LongDoc":"This live view calculates the latency, error rate, and throughput of each distinct parameter set for a given normalized SQL query. Only supports PostgresSQL or MySQL.\n","orgID":"","hidden":false},"px/stirling_errors":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Stirling Error Reporting\nShows the internal error messages and eBPF probe deployment statuses in Stirling.\n'''\nimport px\n\n\ndef stirling_error(start_time: str, source_connector_filter: str, num_head: int):\n df = px.DataFrame(table='stirling_error', start_time=start_time)\n df.node = df.ctx['node']\n\n df = df[px.contains(df.source_connector, source_connector_filter)]\n df = df.head(num_head)\n\n df = df[\"time_\", \"node\", \"source_connector\", \"status\", \"error\", \"context\"]\n return df\n\n\ndef probe_status(start_time: str, tracepoint_filter: str, num_head: int):\n df = px.DataFrame(table=\"probe_status\", start_time=start_time)\n df.node = df.ctx['node']\n\n df = df[px.contains(df.tracepoint, tracepoint_filter)]\n df = df.head(num_head)\n\n df = df[\"time_\", \"node\", \"tracepoint\", \"status\", \"error\", \"info\"]\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now.\",\n \"defaultValue\": \"-7d\"\n },\n {\n \"name\": \"source_connector_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'source_connector' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"tracepoint_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial string to match the 'tracepoint' column.\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"max_num_records\",\n \"type\": \"PX_INT64\",\n \"description\": \"Max number of records to show.\",\n \"defaultValue\": \"1000\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"stirling_error\",\n \"func\": {\n \"name\": \"stirling_error\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"source_connector_filter\",\n \"variable\": \"source_connector_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n },\n {\n \"outputName\": \"probe_status\",\n \"func\": {\n \"name\": \"probe_status\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"tracepoint_filter\",\n \"variable\": \"tracepoint_filter\"\n },\n {\n \"name\": \"num_head\",\n \"variable\": \"max_num_records\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Stirling Component Statuses\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"stirling_error\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"BPF Probe Statuses\",\n \"position\": {\n \"x\": 3,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"globalFuncOutputName\": \"probe_status\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Stirling Error Records","LongDoc":"Shows errors in different Stirling components and deployment statuses of eBPF probes.\n","orgID":"","hidden":false},"px/tracepoint_status":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef tracepoint_info():\n return px.GetTracepointStatus()\n\n\ndef running_tracepoints():\n df = px.GetTracepointStatus()\n df = df[df.state == \"running\"]\n return df\n","vis":"{\n \"variables\": [],\n \"globalFuncs\": [],\n \"widgets\": [\n {\n \"name\": \"Tracepoint Status\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"tracepoint_info\",\n \"args\": []\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"Running tracepoints\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 12,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"running_tracepoints\",\n \"args\": []\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Tracepoint Overview","LongDoc":"Returns information about tracepoints running on the cluster.\n","orgID":"","hidden":false},"px/upids":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef upids_for_namespace(start_time: str, namespace: px.Namespace):\n ''' Gets a list of UPIDs running on `namespace`.\n\n Args:\n @start_time Starting time of the data to examine.\n @namespace: The namespace to filter on.\n '''\n df = px.DataFrame(table='process_stats', start_time=start_time)\n df = df[df.ctx['namespace'] == namespace]\n df.pod = df.ctx['pod_name']\n df.container = df.ctx['container_name']\n df.cmdline = df.ctx['cmdline']\n df = df.groupby(['pod', 'container', 'upid', 'cmdline']).agg()\n df.pod_create_time = px.pod_name_to_start_time(df.pod)\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"namespace\",\n \"type\": \"PX_NAMESPACE\",\n \"description\": \"The name of the namespace for which to list all UPIDs\",\n \"defaultValue\": \"default\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"UPIDs\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 6\n },\n \"func\": {\n \"name\": \"upids_for_namespace\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"namespace\",\n \"variable\": \"namespace\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"UPIDs","LongDoc":"Shows a list of UPIDs running in a given namespace.","orgID":"","hidden":false},"pxbeta/pii_cluster_egress":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport px\n\n\ndef egress_graph(\n start_time: str,\n filter_for_not_ssl: bool,\n use_pixie_dns_resolution: bool,\n):\n \"\"\"Get a graphical representation of all traffic leaving the cluster.\"\"\"\n df = traffic_leaving_cluster(\n start_time,\n filter_for_not_ssl,\n use_pixie_dns_resolution,\n )\n\n df = df.groupby([\"protocol\", \"resolved_domain\"]).agg(\n ssl_sum=(\"ssl\", px.sum),\n count=(\"ssl\", px.count),\n bytes_egressed=(\"bytes_egressed\", px.sum),\n example_req=(\"example_req\", px.any),\n )\n\n df.ssl = px.select(\n df.ssl_sum == 0,\n \"no_ssl\",\n px.select(df.ssl_sum == df.count, \"ssl\", \"both_ssl_and_no_ssl\"),\n )\n # The Graph widget doesn't support categorical labels for edge color,\n # so use a really small number and a really big number to fake a 'latency' for the categorical edge color.\n df.edge_color = px.select(\n df.ssl == \"ssl\", 0, px.select(df.ssl == \"no_ssl\", 201, 101)\n )\n return df\n\n\ndef egress_data(\n start_time: str,\n filter_for_not_ssl: bool,\n use_pixie_dns_resolution: bool,\n):\n \"\"\"Get all traffic leaving the cluster grouped by pod, remote IP address, protocol, and SSL.\"\"\"\n df = traffic_leaving_cluster(\n start_time,\n filter_for_not_ssl,\n use_pixie_dns_resolution,\n )\n\n df = df.groupby([\"protocol\", \"resolved_domain\", \"remote_addr\", \"pod\", \"ssl\"]).agg(\n bytes_egressed=(\"bytes_egressed\", px.sum), example_req=(\"example_req\", px.any)\n )\n\n return df[\n [\n \"resolved_domain\",\n \"protocol\",\n \"remote_addr\",\n \"pod\",\n \"example_req\",\n \"bytes_egressed\",\n \"ssl\",\n ]\n ]\n\n\ndef traffic_leaving_cluster(\n start_time: str,\n filter_for_not_ssl: bool,\n use_pixie_dns_resolution: bool,\n):\n \"\"\"Get all traffic leaving the cluster aggregated by upid,remote_addr pairs.\"\"\"\n df = unified_events_table(start_time)\n\n df = df.groupby(\n [\"node\", \"pod\", \"upid\", \"remote_addr\", \"remote_port\", \"protocol\"]\n ).agg(\n bytes_egressed=(\"bytes\", px.sum),\n example_req=(\"req\", px.any),\n )\n\n # Add whether the connection is ssl'd using conn stats.\n df = ssl_from_conn_stats(df, start_time)\n # Resolve IPs to domains using either nslookup or dns_events.\n df = reverse_lookup_dns(df, start_time, use_pixie_dns_resolution)\n\n # Filter to only not ssl'd if filter_for_not_ssl is set.\n df = df[(not df.ssl) or (not filter_for_not_ssl)]\n\n return df\n\n\ndef ssl_from_conn_stats(df, start_time):\n conn_df = px.DataFrame(\"conn_stats\", start_time=start_time)\n conn_df = conn_df[conn_df.trace_role == 1]\n\n conn_df.cidrs = px.get_cidrs()\n conn_df = conn_df[\n conn_df.remote_addr != \"127.0.0.1\"\n and (conn_df.remote_addr != \"0.0.0.0\" and conn_df.remote_addr != \"-\")\n ]\n conn_df = conn_df[not px.cidrs_contain_ip(conn_df.cidrs, conn_df.remote_addr)]\n\n conn_df.pod = conn_df.ctx[\"pod\"]\n conn_df = conn_df.groupby(\n [\"remote_addr\", \"protocol\", \"remote_port\", \"pod\", \"upid\", \"ssl\"]\n ).agg()\n conn_df.protocol = px.protocol_name(conn_df.protocol)\n\n merged_df = df.merge(\n conn_df,\n how=\"inner\",\n left_on=[\"pod\", \"upid\", \"remote_addr\", \"remote_port\", \"protocol\"],\n right_on=[\"pod\", \"upid\", \"remote_addr\", \"remote_port\", \"protocol\"],\n suffixes=[\"\", \"_y\"],\n )\n return merged_df[\n [\n \"node\",\n \"pod\",\n \"upid\",\n \"remote_addr\",\n \"remote_port\",\n \"protocol\",\n \"ssl\",\n \"bytes_egressed\",\n \"example_req\",\n ]\n ]\n\n\ndef dns_get_answer_at_idx(df, answer_idx):\n df.domain = px.pluck(px.pluck_array(px.pluck(df.req_body, \"queries\"), 0), \"name\")\n df.addr = px.pluck(\n px.pluck_array(px.pluck(df.resp_body, \"answers\"), answer_idx), \"addr\"\n )\n df = df[df.domain != \"\" and df.addr != \"\"]\n df = df[[\"domain\", \"addr\"]]\n return df\n\n\ndef reverse_lookup_dns(df, start_time: str, use_pixie_dns_resolution: bool):\n dns_df = px.DataFrame(\"dns_events\", start_time=start_time)\n\n # Since we don't have an unnest operator, instead we union answers at each index.\n dns_df_merged = dns_get_answer_at_idx(dns_df, 0)\n dns_df_merged = dns_df_merged.append(dns_get_answer_at_idx(dns_df, 1))\n dns_df_merged = dns_df_merged.append(dns_get_answer_at_idx(dns_df, 2))\n dns_df_merged = dns_df_merged.append(dns_get_answer_at_idx(dns_df, 3))\n dns_df_merged = dns_df_merged.append(dns_get_answer_at_idx(dns_df, 4))\n dns_df_merged = dns_df_merged.append(dns_get_answer_at_idx(dns_df, 5))\n\n # Make sure that the mapping from addr -\u003e domain is one-to-many. Should investigate many-to-many situations.\n dns_df_merged = dns_df_merged.groupby([\"addr\"]).agg(\n resolved_domain=(\"domain\", px.any)\n )\n\n df = df.merge(dns_df_merged, how=\"left\", left_on=[\"remote_addr\"], right_on=[\"addr\"])\n df = df.drop([\"addr\"])\n\n df.resolved_domain = px.select(\n (df.resolved_domain == \"\") or (not use_pixie_dns_resolution),\n px.nslookup(df.remote_addr),\n df.resolved_domain,\n )\n return df\n\n\ndef common_events_setup(table_name: str, start_time: str):\n df = px.DataFrame(table_name, start_time=start_time)\n\n df = df[df.trace_role == 1]\n df.cidrs = px.get_cidrs()\n df = df[\n df.remote_addr != \"127.0.0.1\"\n and (df.remote_addr != \"0.0.0.0\" and df.remote_addr != \"-\")\n ]\n df = df[not px.cidrs_contain_ip(df.cidrs, df.remote_addr)]\n\n df.pod = df.ctx[\"pod\"]\n # Get the node from the hostname so that host processes have the correct node.\n df.node = px._exec_hostname()\n return df\n\n\ndef filter_for_pii(df):\n df.req_redacted = px.redact_pii_best_effort(df.req)\n df.contains_pii = px.regex_match(\".*[\u003c]REDACTED[_].*[\u003e].*\", df.req_redacted)\n\n df = df[df.contains_pii]\n return df.drop([\"req_redacted\", \"contains_pii\"])\n\n\ndef unified_events_table(start_time: str):\n out_columns = [\n \"time_\",\n \"upid\",\n \"protocol\",\n \"pod\",\n \"node\",\n \"remote_addr\",\n \"remote_port\",\n \"req\",\n \"req_hdrs\",\n \"resp\",\n \"resp_hdrs\",\n \"bytes\",\n ]\n # HTTP\n http_df = common_events_setup(\"http_events\", start_time)\n http_df.req = http_df.req_body\n http_df.req_hdrs = http_df.req_headers\n http_df.resp = http_df.resp_body\n http_df.resp_hdrs = http_df.resp_headers\n http_df.protocol = \"HTTP\"\n http_df.bytes = http_df.req_body_size\n http_df = filter_for_pii(http_df)\n http_df = http_df[out_columns]\n\n # PGSQL\n pg_df = common_events_setup(\"pgsql_events\", start_time)\n pg_df.req_hdrs = '{\"req_cmd\": \"' + pg_df.req_cmd + '\"}'\n pg_df.resp_hdrs = \"\"\n pg_df.protocol = \"PGSQL\"\n pg_df.bytes = px.length(pg_df.req)\n pg_df = filter_for_pii(pg_df)\n pg_df = pg_df[out_columns]\n\n # MYSQL\n mysql_df = common_events_setup(\"mysql_events\", start_time)\n mysql_df.req_hdrs = '{\"req_cmd\": \"' + px.itoa(mysql_df.req_cmd) + '\"}'\n mysql_df.resp_hdrs = \"\"\n mysql_df.req = mysql_df.req_body\n mysql_df.resp = mysql_df.resp_body\n mysql_df.protocol = \"MySQL\"\n mysql_df.bytes = px.length(mysql_df.req_body)\n mysql_df = filter_for_pii(mysql_df)\n mysql_df = mysql_df[out_columns]\n\n # CQL\n cql_df = common_events_setup(\"cql_events\", start_time)\n cql_df.req_hdrs = '{\"req_op\": \"' + px.itoa(cql_df.req_op) + '\"}'\n cql_df.resp_hdrs = '{\"resp_op\": \"' + px.itoa(cql_df.resp_op) + '\"}'\n cql_df.req = cql_df.req_body\n cql_df.resp = cql_df.resp_body\n cql_df.protocol = \"CQL\"\n cql_df.bytes = px.length(cql_df.req_body)\n cql_df = filter_for_pii(cql_df)\n cql_df = cql_df[out_columns]\n\n # DNS\n dns_df = common_events_setup(\"dns_events\", start_time)\n dns_df.req = dns_df.req_body\n dns_df.req_hdrs = dns_df.req_header\n dns_df.resp = dns_df.resp_body\n dns_df.resp_hdrs = dns_df.resp_header\n dns_df.protocol = \"DNS\"\n dns_df.bytes = px.length(dns_df.req_body)\n dns_df = filter_for_pii(dns_df)\n dns_df = dns_df[out_columns]\n\n # REDIS\n redis_df = common_events_setup(\"redis_events\", start_time)\n redis_df.req_hdrs = '{\"req_cmd\": \"' + redis_df.req_cmd + '\"}'\n redis_df.resp_hdrs = \"\"\n redis_df.req = redis_df.req_args\n redis_df.bytes = px.length(redis_df.req_args)\n redis_df = filter_for_pii(redis_df)\n redis_df = redis_df[out_columns]\n redis_df.protocol = \"Redis\"\n\n df = http_df\n df = df.append(pg_df)\n df = df.append(mysql_df)\n df = df.append(cql_df)\n df = df.append(dns_df)\n df = df.append(redis_df)\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"filter_for_not_ssl\",\n \"type\": \"PX_BOOLEAN\",\n \"description\": \"Include only traffic that is leaving the cluster unencrypted\",\n \"defaultValue\": \"false\"\n },\n {\n \"name\": \"use_pixie_dns_resolution\",\n \"type\": \"PX_BOOLEAN\",\n \"description\": \"Use pixie's DNS data to reverrse IP using forward DNS records, instead of nslookup\",\n \"defaultValue\": \"false\"\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Cluster Egress\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 4\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Graph\",\n \"adjacencyList\": {\n \"fromColumn\": \"protocol\",\n \"toColumn\": \"resolved_domain\"\n },\n \"edgeWeightColumn\": \"bytes_egressed\",\n \"edgeColorColumn\": \"edge_color\",\n \"edgeHoverInfo\": [\n \"resolved_domain\",\n \"protocol\",\n \"bytes_egressed\",\n \"ssl\",\n \"example_req\"\n ],\n \"edgeLength\": 500,\n \"enableDefaultHierarchy\": true,\n \"edgeThresholds\": {\n \"mediumThreshold\": 100,\n \"highThreshold\": 200\n }\n },\n \"func\": {\n \"name\": \"egress_graph\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"filter_for_not_ssl\",\n \"variable\": \"filter_for_not_ssl\"\n },\n {\n \"name\": \"use_pixie_dns_resolution\",\n \"variable\": \"use_pixie_dns_resolution\"\n }\n ]\n }\n },\n {\n \"name\": \"Egress by pod, IP\",\n \"position\": {\n \"x\": 0,\n \"y\": 4,\n \"w\": 12,\n \"h\": 4\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n },\n \"func\": {\n \"name\": \"egress_data\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"filter_for_not_ssl\",\n \"variable\": \"filter_for_not_ssl\"\n },\n {\n \"name\": \"use_pixie_dns_resolution\",\n \"variable\": \"use_pixie_dns_resolution\"\n }\n ]\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Traffic containing PII leaving the cluster","LongDoc":"This view displays a summary of traffic containing PII from the cluster to external endpoints.\n","orgID":"","hidden":false},"pxbeta/service_endpoint":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Endpoint Overview\n\n This script gets an overview of an individual endpoint on an\n individual service, summarizing its request statistics.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = False\n# Flag to filter out health checks from the data.\nfilter_health_checks = False\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = False\n\n\ndef endpoint_let_timeseries(start_time: str, service: px.Service, endpoint: str):\n ''' Compute the let as a timeseries for requests received by `service` at `endpoint`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @service: The name of the service to filter on.\n @endpoint: The endpoint to filter on.\n\n '''\n df = let_helper(start_time, service)\n df = df[px._match_endpoint(df.req_path, endpoint)]\n\n df = df.groupby(['timestamp']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate_per_window=('failure', px.mean),\n throughput_total=('latency', px.count),\n bytes_total=('resp_body_size', px.sum)\n )\n\n # Format the result of LET aggregates into proper scalar formats and\n # time series.\n df.latency_p50 = px.DurationNanos(\n px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(\n px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(\n px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n df.request_throughput = df.throughput_total / window_ns\n df.errors_per_ns = df.error_rate_per_window * \\\n df.request_throughput / px.DurationNanos(1)\n df.error_rate = px.Percent(df.error_rate_per_window)\n df.bytes_per_ns = df.bytes_total / window_ns\n df.time_ = df.timestamp\n\n return df[['time_', 'latency_p50', 'latency_p90', 'latency_p99',\n 'request_throughput', 'errors_per_ns', 'error_rate', 'bytes_per_ns']]\n\n\ndef endpoint_slow_requests(start_time: str, service: px.Service, endpoint: str):\n df = let_helper(start_time, service)\n df = df[px._match_endpoint(df.req_path, endpoint)]\n df = add_source_dest_columns(df)\n df = add_source_dest_links(df, start_time)\n\n quantiles = df.groupby('service').agg(\n latency_quantiles=('latency', px.quantiles)\n )\n quantiles.service_p99 = px.pluck_float64(\n quantiles.latency_quantiles, 'p99')\n quantiles = quantiles.drop('latency_quantiles')\n requests = df.merge(quantiles, left_on='service', right_on='service', how='inner',\n suffixes=['', '_x'])\n requests = requests[requests.latency \u003e= px.floor(requests.service_p99)]\n return requests[['time_', 'source', 'latency', 'req_method',\n 'req_path', 'req_body', 'resp_status', 'resp_body']].head(100)\n\n\ndef let_helper(start_time: str, service: px.Service):\n ''' Compute the initial part of the let for requests.\n\n Args:\n @start_time: The timestamp of data to start at.\n @service: The service to filter on.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.trace_role == 2]\n\n df.service = df.ctx['service']\n df = df[px.contains(df.service, service)]\n df.pod = df.ctx['pod']\n df.timestamp = px.bin(df.time_, window_ns)\n df.failure = df.resp_status \u003e= 400\n\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n df = df[filter_out_conds]\n\n return df\n\n\ndef add_source_dest_columns(df):\n ''' Add source and destination columns.\n\n Connections are traced server-side (trace_role==2), unless the server is outside\n of the cluster in which case the connection is traced client-side (trace_role==1).\n\n With trace_role==2: source is the `remote_addr` col, destination is the `pod` col.\n With trace_role==1: source is the `pod` col, destination is the `remote_addr` col.\n\n Input DataFrame must contain trace_role, upid, remote_addr columns.\n '''\n df.pod = df.ctx['pod']\n df.namespace = df.ctx['namespace']\n\n # If remote_addr is a pod, get its name. If not, use IP address.\n df.ra_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n df.is_ra_pod = df.ra_pod != ''\n df.ra_name = px.select(df.is_ra_pod, df.ra_pod, df.remote_addr)\n\n df.is_server_tracing = df.trace_role == 2\n df.is_source_pod_type = px.select(df.is_server_tracing, df.is_ra_pod, True)\n df.is_dest_pod_type = px.select(df.is_server_tracing, True, df.is_ra_pod)\n\n # Set source and destination based on trace_role.\n df.source = px.select(df.is_server_tracing, df.ra_name, df.pod)\n df.destination = px.select(df.is_server_tracing, df.pod, df.ra_name)\n\n # Filter out messages with empty source / destination.\n df = df[df.source != '']\n df = df[df.destination != '']\n\n df = df.drop(['ra_pod', 'is_ra_pod', 'ra_name', 'is_server_tracing'])\n\n return df\n\n\ndef add_source_dest_links(df, start_time: str):\n ''' Modifies the source and destination columns to display deeplinks in the UI.\n Clicking on a pod name in either column will run the px/pod script for that pod.\n Clicking on an IP address, will run the px/net_flow_graph script showing all\n network connections to/from that address.\n\n Input DataFrame must contain source, destination, is_source_pod_type,\n is_dest_pod_type, and namespace columns.\n '''\n\n # Source linking. If source is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.src_pod_link = px.script_reference(df.source, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.source\n })\n df.src_link = px.script_reference(df.source, 'px/net_flow_graph', {\n 'start_time': start_time,\n 'namespace': df.namespace,\n 'from_entity_filter': df.source,\n 'to_entity_filter': '',\n 'throughput_filter': '0.0'\n })\n df.source = px.select(df.is_source_pod_type, df.src_pod_link, df.src_link)\n\n # If destination is a pod, link to px/pod. If an IP addr, link to px/net_flow_graph.\n df.dest_pod_link = px.script_reference(df.destination, 'px/pod', {\n 'start_time': start_time,\n 'pod': df.destination\n })\n df.dest_link = px.script_reference(df.destination, 'px/net_flow_graph', {\n 'start_time': start_time,\n 'namespace': df.namespace,\n 'from_entity_filter': '',\n 'to_entity_filter': df.destination,\n 'throughput_filter': '0.0'\n })\n df.destination = px.select(df.is_dest_pod_type, df.dest_pod_link, df.dest_link)\n\n df = df.drop(['src_pod_link', 'src_link', 'is_source_pod_type', 'dest_pod_link',\n 'dest_link', 'is_dest_pod_type'])\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"service\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The name of the service to get stats for. Format: ns/svc_name\",\n \"defaultValue\": \"\"\n },\n {\n \"name\": \"endpoint\",\n \"type\": \"PX_STRING\",\n \"description\": \"The endpoint to look at stats for. Format: /path/to/endpoint/*/more/path \",\n \"defaultValue\": \"/\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"endpoint_let_timeseries\",\n \"func\": {\n \"name\": \"endpoint_let_timeseries\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n },\n {\n \"name\": \"endpoint\",\n \"variable\": \"endpoint\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"HTTP Requests\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"request throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Errors\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"errors_per_ns\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"error rate\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p90\",\n \"mode\": \"MODE_LINE\"\n },\n {\n \"value\": \"latency_p99\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Response Data Throughput\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let_timeseries\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"bytes_per_ns\",\n \"mode\": \"MODE_LINE\"\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Response data throughput\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"Sample of Slow Requests\",\n \"position\": {\n \"x\": 4,\n \"y\": 6,\n \"w\": 8,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"endpoint_slow_requests\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n },\n {\n \"name\": \"endpoint\",\n \"variable\": \"endpoint\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Endpoint Overview","LongDoc":"This script gets an overview of an individual endpoint for an individual service, summarizing its request statistics.\n","orgID":"","hidden":false},"pxbeta/service_endpoints":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Endpoints Overview\n\n This script gets an overview of all the endpoints in `service`.\n'''\nimport px\n\nns_per_ms = 1000 * 1000\nns_per_s = 1000 * ns_per_ms\n# Window size to use on time_ column for bucketing.\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n# Flag to filter out requests that come from an unresolvable IP.\nfilter_unresolved_inbound = False\n# Flag to filter out health checks from the data.\nfilter_health_checks = False\n# Flag to filter out ready checks from the data.\nfilter_ready_checks = False\n\n\ndef endpoints(start_time: str, service: px.Service):\n ''' Get a list of the endpoints in `service`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @service: The service to filter on.\n\n '''\n df = endpoint_let_helper(start_time, service)\n df = request_path_endpoint_clustering(df)\n df = df.groupby(['endpoint']).agg()\n df.endpoint = px.script_reference(df.endpoint, 'pxbeta/service_endpoint', {\n 'start_time': start_time,\n 'service': service,\n 'endpoint': df.endpoint,\n })\n return df\n\n\ndef endpoint_let(start_time: str, service: px.Service):\n ''' Compute the let as a timeseries for requests received by endpoints in `service`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @service: The service to filter on.\n\n '''\n\n # Calculate LET for each svc edge in the svc graph over each time window.\n # Each edge starts at a requester ('remote_addr') and ends at a\n # responder service.\n\n df = endpoint_let_helper(start_time, service)\n df = request_path_endpoint_clustering(df)\n\n df = df.groupby(['timestamp', 'endpoint']).agg(\n latency_quantiles=('latency', px.quantiles),\n error_rate=('failure', px.mean),\n throughput_total=('latency', px.count),\n inbound_bytes_total=('req_body_size', px.sum),\n outbound_bytes_total=('resp_body_size', px.sum)\n )\n\n df.latency_p50 = px.DurationNanos(\n px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n df.latency_p90 = px.DurationNanos(\n px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n df.latency_p99 = px.DurationNanos(\n px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n\n df.request_throughput = df.throughput_total / window_ns\n df.inbound_throughput = df.inbound_bytes_total / window_ns\n df.outbound_throughput = df.outbound_bytes_total / window_ns\n df.error_rate = px.Percent(df.error_rate)\n df.time_ = df.timestamp\n\n return df[['time_', 'endpoint', 'latency_p50', 'latency_p90',\n 'latency_p99', 'request_throughput', 'error_rate',\n 'inbound_throughput', 'outbound_throughput']]\n\n\ndef endpoint_let_helper(start_time: str, service: px.Service):\n ''' Compute the let as a timeseries for requests received or by services in `namespace`.\n\n Args:\n @start_time: The timestamp of data to start at.\n @namespace: The namespace to filter on.\n @groupby_cols: The columns to group on.\n\n '''\n df = px.DataFrame(table='http_events', start_time=start_time)\n\n # Filter only to inbound service traffic (server-side).\n # Don't include traffic initiated by this service to an external location.\n df = df[df.trace_role == 2]\n\n df.service = df.ctx['service']\n df = df[px.contains(df.service, service)]\n df.timestamp = px.bin(df.time_, window_ns)\n df.failure = df.resp_status \u003e= 400\n\n filter_out_conds = ((df.req_path != '/healthz' or not filter_health_checks) and (\n df.req_path != '/readyz' or not filter_ready_checks)) and (\n df['remote_addr'] != '-' or not filter_unresolved_inbound)\n df = df[filter_out_conds]\n\n return df\n\n\ndef request_path_endpoint_clustering(let_df):\n clustering_df = let_df.agg(\n clustering=(\"req_path\", px._build_request_path_clusters)\n )\n merged_df = let_df.merge(\n clustering_df, how=\"outer\", right_on=[], left_on=[], suffixes=[\"\", \"\"]\n )\n merged_df.endpoint = px._predict_request_path_cluster(\n merged_df.req_path, merged_df.clustering\n )\n return merged_df.drop([\"clustering\"])\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The relative start time of the window. Current time is assumed to be now\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"service\",\n \"type\": \"PX_SERVICE\",\n \"description\": \"The name of the service to get stats for.\",\n \"defaultValue\": \"default\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"endpoint_let\",\n \"func\": {\n \"name\": \"endpoint_let\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Endpoints\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"func\": {\n \"name\": \"endpoints\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"service\",\n \"variable\": \"service\"\n }\n ]\n },\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n },\n {\n \"name\": \"HTTP Request Throughput\",\n \"position\": {\n \"x\": 4,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"request_throughput\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"endpoint\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"requests per second\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP Request Error Rate\",\n \"position\": {\n \"x\": 8,\n \"y\": 0,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"error_rate\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"endpoint\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"error rate (%)\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP P50 Latency\",\n \"position\": {\n \"x\": 0,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p50\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"endpoint\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Mean latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP P90 Latency\",\n \"position\": {\n \"x\": 4,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p90\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"endpoint\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Mean latency\"\n },\n \"xAxis\": null\n }\n },\n {\n \"name\": \"HTTP P99 Latency\",\n \"position\": {\n \"x\": 8,\n \"y\": 3,\n \"w\": 4,\n \"h\": 3\n },\n \"globalFuncOutputName\": \"endpoint_let\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.TimeseriesChart\",\n \"timeseries\": [\n {\n \"value\": \"latency_p99\",\n \"mode\": \"MODE_LINE\",\n \"series\": \"endpoint\",\n \"stackBySeries\": false\n }\n ],\n \"title\": \"\",\n \"yAxis\": {\n \"label\": \"Mean latency\"\n },\n \"xAxis\": null\n }\n }\n ]\n}\n","placement":"","ShortDoc":"Endpoints Overview","LongDoc":"This script gets an overview of the endpoints for a service, summarizing their request statistics.\n","orgID":"","hidden":false},"pxbeta/vfs_snoop":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\nimport pxtrace\nimport px\n\nprogram = \"\"\"\n#include \u003clinux/sched.h\u003e\n#include \u003clinux/file.h\u003e\n#include \u003clinux/fs.h\u003e\n#include \u003clinux/dcache.h\u003e\n\nkprobe:vfs_write\n{\n $file = (struct file*)arg0;\n\n $dentry = $file-\u003ef_path.dentry;\n $filename = str($dentry-\u003ed_name.name);\n\n $parent1_dentry = $dentry-\u003ed_parent;\n $parent1_filename = str($parent1_dentry-\u003ed_name.name);\n\n $parent2_dentry = $parent1_dentry-\u003ed_parent;\n $parent2_filename = str($parent2_dentry-\u003ed_name.name);\n\n printf(\\\"time_:%d pid:%d pid_start_time:%d op:%s filename:%s d1:%s d2:%s\\\",\n nsecs,\n pid,\n ((struct task_struct*)curtask)-\u003egroup_leader-\u003estart_time / 10000000,\n \"write\",\n $filename,\n $parent1_filename,\n $parent2_filename);\n}\n\nkprobe:vfs_unlink\n{\n $dentry = (struct dentry*)arg1;\n $filename = str($dentry-\u003ed_name.name);\n\n $parent1_dentry = $dentry-\u003ed_parent;\n $parent1_filename = str($parent1_dentry-\u003ed_name.name);\n\n $parent2_dentry = $parent1_dentry-\u003ed_parent;\n $parent2_filename = str($parent2_dentry-\u003ed_name.name);\n\n printf(\\\"time_:%d pid:%d pid_start_time:%d op:%s filename:%s d1:%s d2:%s\\\",\n nsecs,\n pid,\n ((struct task_struct*)curtask)-\u003egroup_leader-\u003estart_time / 10000000,\n \"delete\",\n $filename,\n $parent1_filename,\n $parent2_filename);\n}\n\"\"\"\n\n\ndef add_pod_name(df):\n proc = px.DataFrame(table='process_stats')\n proc.pod = proc.ctx['pod']\n proc.pid = px.upid_to_pid(proc.upid)\n proc.asid = px.upid_to_asid(proc.upid)\n proc = proc.groupby(['pod', 'pid', 'asid']).agg()\n\n df = df.merge(proc,\n how='left',\n left_on=['pid', 'asid'],\n right_on=['pid', 'asid'],\n suffixes=['', '_x'])\n\n return df\n\n\ndef exec_snoop_func(path_substr: str):\n table_name = 'vfs_table'\n pxtrace.UpsertTracepoint('vfs_tracer',\n table_name,\n program,\n pxtrace.kprobe(),\n \"5m\")\n df = px.DataFrame(table=table_name)\n\n # Grab the node from exec_hostname instead of `ctx`, because not all PIDs have a pod.\n df.node = px._exec_hostname()\n\n # Try to add the pod name, if available. If pod is not matched, the field will be blank.\n df.asid = px.asid()\n df = add_pod_name(df)\n\n # Filter to paths that are two-levels only (i.e. /dir/file)\n df = df[df.d2 == \"/\" and not df.d1 == \"/\"]\n\n # Build the path.\n df.path = \"/\" + df.d1 + \"/\" + df.filename\n df = df.drop(['filename', 'd1', 'd2'])\n\n # Filter for the requested commands.\n df = df[px.contains(df.path, path_substr)]\n\n return df['time_', 'node', 'pid', 'pod', 'op', 'path']\n","vis":" {\n \"variables\": [\n {\n \"name\": \"path_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The full/partial name of the file path on which to filter.\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"results\",\n \"func\": {\n \"name\": \"exec_snoop_func\",\n \"args\": [\n {\n \"name\": \"path_substr\",\n \"variable\": \"path_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": [\n {\n \"name\": \"Results\",\n \"position\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 5\n },\n \"globalFuncOutputName\": \"results\",\n \"displaySpec\": {\n \"@type\": \"types.px.dev/px.vispb.Table\"\n }\n }\n ]\n}\n","placement":"","ShortDoc":"VFS Write/Delete Snoop","LongDoc":"Traces file writes and deletions.","orgID":"","hidden":false},"sotw/dns_external_fqdn_list":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' DNS Traffic FQDN List\nLists external, fully qualified domain names (FQDNs) from all\nDNS traffic on the cluster for a specified amount of time.\n'''\nimport px\n\n\ndef fqdn_list(start_time: str):\n\n # The dns_events table pairs DNS requests with their responses.\n df = px.DataFrame(table='dns_events', start_time=start_time)\n\n # Parse the DNS response to determine if it was successfully resolved.\n df.resp_body = px.pluck(df.resp_body, 'answers')\n df.request_resolved = px.contains(df.resp_body, 'name')\n\n # Filter for resolved requests.\n df = df[df.request_resolved]\n\n # Parse the DNS request for query name.\n # TODO: cleanup this logic when we support object types.\n df.req_body = px.pluck(df.req_body, 'queries')\n df.idx1 = px.find(df.req_body, '\\\"name\\\":')\n df.idz = px.length(df.req_body) - (df.idx1 + 8) - 3\n df.fqdn_partial = px.substring(df.req_body, df.idx1 + 8, df.idz)\n df.idx2 = px.find(df.fqdn_partial, ',')\n df.fqdn = px.substring(df.fqdn_partial, 0, df.idx2 - 1)\n\n # Filter out local domain queries:\n # If your k8s cluster uses a different internal domain suffix, add it here.\n df = df[not px.contains(df.fqdn, '.local')]\n df = df[not px.contains(df.fqdn, '.internal')]\n\n # Add link to script that will show all requests for specific query.\n df.link = px.script_reference('All DNS requests containing FQDN as substring',\n 'sotw/dns_queries_filtered', {\n 'start_time': start_time,\n 'query_name_filter': df.fqdn\n })\n\n # Group by (fqdn, link) and count number of requests.\n df = df.groupby(['fqdn', 'link']).agg(\n num_requests=('request_resolved', px.count)\n )\n\n return df\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"fqdn_list\",\n \"func\": {\n \"name\": \"fqdn_list\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n }\n ]\n }\n }\n ],\n \"widgets\": []\n}\n","placement":"","ShortDoc":"Lists external FQDNs from DNS traffic.","LongDoc":"Lists external, fully qualified domain names (FQDNs) from all DNS traffic on the cluster for a specified amount of time.","orgID":"","hidden":false},"sotw/dns_queries_filtered":{"pxl":"# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' DNS Queries filtered by query name.\nLists all DNS queries filtered by a specific query name.\n'''\nimport px\n\n\ndef dns_requests(start_time: str, query_name_filter: str):\n\n # The dns_events table pairs DNS requests with their responses.\n df = px.DataFrame(table='dns_events', start_time=start_time)\n\n # Add context.\n df.pod = df.ctx['pod']\n\n # Convert DNS IP to string.\n df.dns_server = px.nslookup(df.remote_addr)\n\n # Parse the DNS response to determine if it was successfully resolved.\n df.resp_body = px.pluck(df.resp_body, 'answers')\n df.request_resolved = px.contains(df.resp_body, 'name')\n\n # Parse the DNS request for query name.\n # TODO: cleanup this logic when we support object types.\n df.req_body = px.pluck(df.req_body, 'queries')\n df.idx1 = px.find(df.req_body, '\\\"name\\\":')\n df.idz = px.length(df.req_body) - (df.idx1 + 8) - 3\n df.query_name_partial = px.substring(df.req_body, df.idx1 + 8, df.idz)\n df.idx2 = px.find(df.query_name_partial, ',')\n df.dns_query_name = px.substring(df.query_name_partial, 0, df.idx2 - 1)\n df = df.drop(['idx1', 'idx2'])\n\n # Group all unique pod / dns_server pairings containing query_name_filter as substring.\n df = df[px.contains(df.dns_query_name, query_name_filter)]\n df = df.groupby(['pod', 'dns_server', 'dns_query_name']).agg(\n num_requests=('request_resolved', px.count),\n resolved=('request_resolved', px.mean),\n latency=('latency', px.quantiles)\n )\n\n df.resolved = px.Percent(df.resolved)\n\n return df[['pod', 'dns_server', 'dns_query_name', 'num_requests', 'resolved', 'latency']]\n","vis":"{\n \"variables\": [\n {\n \"name\": \"start_time\",\n \"type\": \"PX_STRING\",\n \"description\": \"The start time of the window in time units before now.\",\n \"defaultValue\": \"-5m\"\n },\n {\n \"name\": \"query_name_filter\",\n \"type\": \"PX_STRING\",\n \"description\": \"The partial query name used to filter the DNS requests by (e.g. your.domain.name).\",\n \"defaultValue\": \"\"\n }\n ],\n \"globalFuncs\": [\n {\n \"outputName\": \"dns_requests\",\n \"func\": {\n \"name\": \"dns_requests\",\n \"args\": [\n {\n \"name\": \"start_time\",\n \"variable\": \"start_time\"\n },\n {\n \"name\": \"query_name_filter\",\n \"variable\": \"query_name_filter\"\n }\n ]\n }\n }\n ],\n \"widgets\": []\n}\n","placement":"","ShortDoc":"All DNS queries filtered by a specific query name.","LongDoc":"Lists all DNS queries filtered by a specific query name.","orgID":"","hidden":false}}}