一、背景

使用ebpf 监控grpc-go的应用，grpc-go http2 client的处理点

func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) error {
......
}// operateHeaders takes action on the decoded headers.
func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
}

使用ebpf 监控埋点:

SEC("uprobe/google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
int uprobe__probe_http2_client_operate_headers(struct pt_regs* ctx) {
}

运行流程:

我们在 uprobe__probe_http2_client_operate_headers 里做一些处理，统计grpc的数据，进行遥测。编写代码过程中我们需要关注的一些事情

1、golang 程序中 t *http2Client 和  frame *http2.MetaHeadersFrame 在寄存器中的位置是如何分配的？

type MetaHeadersFrame struct {*HeadersFrame// Fields are the fields contained in the HEADERS and// CONTINUATION frames. The underlying slice is owned by the// Framer and must not be retained after the next call to// ReadFrame.//// Fields are guaranteed to be in the correct http2 order and// not have unknown pseudo header fields or invalid header// field names or values. Required pseudo header fields may be// missing, however. Use the MetaHeadersFrame.Pseudo accessor// method access pseudo headers.Fields []hpack.HeaderField// Truncated is whether the max header list size limit was hit// and Fields is incomplete. The hpack decoder state is still// valid, however.Truncated bool
}

2、http2Client 和 MetaHeadersFrame 结构体中各个成员的偏移量如何确定？

type http2Client struct {lastRead  int64 // Keep this field 64-bit aligned. Accessed atomically.ctx       context.Contextcancel    context.CancelFuncctxDone   <-chan struct{} // Cache the ctx.Done() chan.userAgent string// address contains the resolver returned address for this transport.// If the `ServerName` field is set, it takes precedence over `CallHdr.Host`// passed to `NewStream`, when determining the :authority header.address    resolver.Addressmd         metadata.MD
.....
}

 以 MetaHeadersFrame 为例子

 我们在开发中需要找到 正确的offset 才能正确的取到内存。

二、我们需要做什么？

1、如何确定参数在寄存器中的位置？

我们使用llvm-dwarfdump-14 查看二进制中dwarf 构成:

llvm-dwarfdump-14 grpc-client |grep "loopyWriter).writeHeader" -C 100

返回格式:

0x0018c375:   DW_TAG_subprogramDW_AT_name	("google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")DW_AT_low_pc	(0x00000000007139a0)DW_AT_high_pc	(0x0000000000713df2)DW_AT_frame_base	(DW_OP_call_frame_cfa)DW_AT_decl_file	("/home/zhanglei/data/grpc-demo/vendor/google.golang.org/grpc/internal/transport/controlbuf.go")DW_AT_external	(0x01)0x0018c3d2:     DW_TAG_formal_parameterDW_AT_name	("l")DW_AT_variable_parameter	(0x00)DW_AT_decl_line	(677)DW_AT_type	(0x00000000000e6bc2 "google.golang.org/grpc/internal/transport.loopyWriter *")DW_AT_location	(0x001a603f: [0x00000000007139a0, 0x00000000007139fc): DW_OP_reg0 RAX[0x00000000007139fc, 0x0000000000713df2): DW_OP_call_frame_cfa)0x0018c3e0:     DW_TAG_formal_parameterDW_AT_name	("streamID")DW_AT_variable_parameter	(0x00)DW_AT_decl_line	(677)DW_AT_type	(0x0000000000082ad4 "uint32")DW_AT_location	(0x001a6085: [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg3 RBX[0x0000000000713a02, 0x0000000000713df2): DW_OP_fbreg +8)0x0018c3f5:     DW_TAG_formal_parameterDW_AT_name	("endStream")DW_AT_variable_parameter	(0x00)DW_AT_decl_line	(677)DW_AT_type	(0x0000000000082b87 "bool")DW_AT_location	(0x001a60cc: [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg2 RCX[0x0000000000713a02, 0x0000000000713df2): DW_OP_fbreg +12)0x0018c40b:     DW_TAG_formal_parameterDW_AT_name	("hf")DW_AT_variable_parameter	(0x00)DW_AT_decl_line	(677)DW_AT_type	(0x00000000000e2826 "[]golang.org/x/net/http2/hpack.HeaderField")DW_AT_location	(0x001a6113: [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x8, DW_OP_reg8 R8, DW_OP_piece 0x8[0x0000000000713a02, 0x0000000000713a4e): DW_OP_fbreg +16, DW_OP_piece 0x8, DW_OP_fbreg +24, DW_OP_piece 0x8, DW_OP_piece 0x8[0x0000000000713a4e, 0x0000000000713df2): DW_OP_piece 0x8, DW_OP_fbreg +24, DW_OP_piece 0x8, DW_OP_piece 0x8)0x0018c41a:     DW_TAG_formal_parameterDW_AT_name	("onWrite")DW_AT_variable_parameter	(0x00)DW_AT_decl_line	(677)DW_AT_type	(0x00000000000844f2 "func()")DW_AT_location	(0x001a6184: [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg9 R9)

数据格式简介:

我们可以找到func (l *loopyWriter) writeHeader 的参数的所在寄存器的位置

DW_AT_name	("l")  的 [0x00000000007139a0, 0x00000000007139fc): DW_OP_reg0 RAX 在寄存器0 上
DW_AT_name	("streamID") : DW_OP_reg3 RBX 在寄存器3上
DW_AT_name	("endStream")  DW_OP_reg2 RCX 在寄存器2上
DW_AT_name	("hf") 在寄存器5 上 DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x8, DW_OP_reg8 R8, DW_OP_piece 0x8
DW_AT_name	("onWrite")  DW_OP_reg9 R9 在寄存器9上

2、如何定位结构体中的成员

0x000e6c07:   DW_TAG_structure_typeDW_AT_name	("google.golang.org/grpc/internal/transport.loopyWriter")DW_AT_byte_size	(88)DW_AT_GO_kind	(0x19)DW_AT_GO_runtime_type	(0x00000000000b9a20)0x000e6c48:     DW_TAG_memberDW_AT_name	("side")DW_AT_data_member_location	(0)DW_AT_type	(0x00000000000e5e96 "google.golang.org/grpc/internal/transport.side")DW_AT_GO_embedded_field	(0x00)0x000e6c54:     DW_TAG_memberDW_AT_name	("cbuf")DW_AT_data_member_location	(8)DW_AT_type	(0x00000000000e681d "google.golang.org/grpc/internal/transport.controlBuffer *")DW_AT_GO_embedded_field	(0x00)0x000e6c60:     DW_TAG_memberDW_AT_name	("sendQuota")DW_AT_data_member_location	(16)DW_AT_type	(0x0000000000082ad4 "uint32")DW_AT_GO_embedded_field	(0x00)0x000e6c71:     DW_TAG_memberDW_AT_name	("oiws")DW_AT_data_member_location	(20)DW_AT_type	(0x0000000000082ad4 "uint32")DW_AT_GO_embedded_field	(0x00)0x000e6c7d:     DW_TAG_memberDW_AT_name	("estdStreams")DW_AT_data_member_location	(24)DW_AT_type	(0x00000000000e6d3c "map[uint32]*google.golang.org/grpc/internal/transport.outStream")DW_AT_GO_embedded_field	(0x00)0x000e6c90:     DW_TAG_memberDW_AT_name	("activeStreams")DW_AT_data_member_location	(32)DW_AT_type	(0x00000000000e6eaa "google.golang.org/grpc/internal/transport.outStreamList *")DW_AT_GO_embedded_field	(0x00)0x000e6ca5:     DW_TAG_memberDW_AT_name	("framer")DW_AT_data_member_location	(40)DW_AT_type	(0x00000000000e6f8a "google.golang.org/grpc/internal/transport.framer *")DW_AT_GO_embedded_field	(0x00)

 以 loopyWriter 结构体为例子:

type loopyWriter struct {side      sidecbuf      *controlBuffersendQuota uint32oiws      uint32 // outbound initial window size.// estdStreams is map of all established streams that are not cleaned-up yet.// On client-side, this is all streams whose headers were sent out.// On server-side, this is all streams whose headers were received.estdStreams map[uint32]*outStream // Established streams.// activeStreams is a linked-list of all streams that have data to send and some// stream-level flow control quota.// Each of these streams internally have a list of data items(and perhaps trailers// on the server-side) to be sent out.activeStreams *outStreamListframer        *framerhBuf          *bytes.Buffer  // The buffer for HPACK encoding.hEnc          *hpack.Encoder // HPACK encoder.bdpEst        *bdpEstimatordraining      bool// Side-specific handlersssGoAwayHandler func(*goAway) (bool, error)
}

side 的偏移量是  0 

cbuf 是 8 

sendQuota 是 16 

oiws 是 20

estdStreams 是 24

activeStreams 是 32 

framer 是 40

3、如何确定接口类型地址

nm grpc-client|grep "TCPConn,net.Conn"

000000000093d280 R go.itab.*net.TCPConn,net.Conn

 发现对应的符号表地址:

nm grpc-server|grep "TCPConn,net.Conn"
000000000093bfc0 R go.itab.*net.TCPConn,net.Conn

4、使用gdb 证明这些数据

(gdb) i args
l = 0xc00011c060
streamID = 1
endStream = false
hf = {array = 0xc00007e870, len = 2, cap = 2}
onWrite = {void (void)} 0x0
~r0 = 
(gdb) i r
rax            0xc00011c060        824634884192
rbx            0x1                 1
rcx            0x0                 0
rdx            0xc0001b29c0        824635500992
rsi            0x2                 2
rdi            0xc00007e870        824634239088
rbp            0xc000129ec8        0xc000129ec8
rsp            0xc000129e78        0xc000129e78
r8             0x2                 2
r9             0xc000021f80        824633859968
r10            0xc00007e8c0        824634239168
r11            0x1                 1
r12            0x1                 1
r13            0xffffffffffffffff  -1
r14            0xc00019d520        824635413792
r15            0x0                 0
rip            0x7160c0            0x7160c0 
eflags         0x246               [ PF ZF IF ]
cs             0x33                51
ss             0x2b                43
ds             0x0                 0
es             0x0                 0
fs             0x0                 0
gs             0x0                 0

发现l 位于寄存器 

l -> rax

streamID -> rbx

endStream -> rcx

hf -> rdi

接口类型:

(gdb) p (*l.framer.writer).conn
$8 = {tab = 0x93bfc0 , data = 0xc000010290}

 发现对应的符号表地址:

nm grpc-server|grep "TCPConn,net.Conn"
000000000093bfc0 R go.itab.*net.TCPConn,net.Conn

三、发现golang应用

uprober 包含了 多个rule

每个rule 又包含Selector和 Checker

Selector 用来根据版本选择对应的Hooker

Checker 用来检查 动态库或者二进制文件是否满足匹配规则

运行流程：

四、golang 注册ebpf 钩子

五.使用偏移量获取golang变量

定义 location 结构体:

typedef struct {__s64 stack_offset; // 栈上的偏移量__s64 _register; // 寄存器的偏移量__u8 in_register; //是否在寄存器上__u8 exists;
} location_t;// golang 切片
typedef struct {location_t ptr;location_t len;location_t cap;
} slice_location_t;

golang 参数变量在寄存器的布局:

// This function was adapted from https://github.com/go-delve/delve:
// - https://github.com/go-delve/delve/blob/cd9e6c02a6ca5f0d66c1f770ee10a0d8f4419333/pkg/proc/internal/ebpf/bpf/trace.bpf.c#L43
// which is licensed under MIT.
static __always_inline int read_register(struct pt_regs* ctx, int64_t regnum, void* dest) {// This volatile temporary variable is need when building with clang-14,// or the verifier will complain that we dereference a modified context// pointer.//// What happened in this case, is that the compiler tried to be smart by// incrementing the context pointer, before jumping to code that will// copy the value pointed to by the new pointer to `dest`. The generated// code looked like this:////      r1 += 40           // Increment the ptr//      goto +3    // goto __builtin_memcpy//// What the memcpy does is deference the resulting pointer to get the// CPU register value (that’s where the bug was), then put it in the// dest location:////      r1 = *(u64 *)(r1 + 0)  // BUG: Get the register value.//                             // This is the "modified context pointer"//      *(u64 *)(r3 + 0) = r1  // Put it in dest//// By incrementing the pointer before dereferencing it, the verifier no// longer considering r1 to be a pointer to the context, but as a// pointer to some random memory address (even though it is in the// memory the range of the context struct).//// What we want the compiler to generate is something like this:////      // Switch branch://      r1 = *(u64 *)(r1 + 40) // read value to tmp var//      goto +30      // goto *dest = tmp////      // *dest = tmp//      *(u64 *)(r3 + 0) = r1//// This volatile `tmp` variable makes the compiler generate the code above.volatile u64 tmp = 0;switch (regnum) {case 0: // RAXtmp = ctx->ax;break;case 1: // RDXtmp = ctx->dx;break;case 2: // RCXtmp = ctx->cx;break;case 3: // RBXtmp = ctx->bx;break;case 4: // RSItmp = ctx->si;break;case 5: // RDItmp = ctx->di;break;case 6: // RBPtmp = ctx->bp;break;case 7: // RSPtmp = ctx->sp;break;case 8: // R8tmp = ctx->r8;break;case 9: // R9tmp = ctx->r9;break;case 10: // R10tmp = ctx->r10;break;case 11: // R11tmp = ctx->r11;break;case 12: // R12tmp = ctx->r12;break;case 13: // R13tmp = ctx->r13;break;case 14: // R14tmp = ctx->r14;break;case 15: // R15tmp = ctx->r15;break;default:return 1;}*(u64*)dest = tmp;return 0;
}

 读取偏移量：

static __always_inline int read_location(struct pt_regs* ctx, location_t* loc, size_t size, void* dest) {if (!loc->exists) {return 0;}if (loc->in_register) {if (size != REG_SIZE) {return 1;}return read_register(ctx, loc->_register, dest);} else {return read_stack(ctx, loc->stack_offset, size, dest);}
}

static __always_inline int read_stack(struct pt_regs* ctx, int64_t stack_offset, size_t size, void* dest) {// `ctx->sp` is correct for both x86_64 and ARM64uintptr_t stack_pointer = (uintptr_t) ctx->sp;uintptr_t address = stack_pointer + stack_offset;return bpf_probe_read_user(dest, size, (void*) address);
}

举一个例子:

SEC("uprobe/google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
int uprobe__probe_loopy_writer_write_header(struct pt_regs* ctx) {return 0;
}

 从map读取偏移量:

uint32_t pid = bpf_get_current_pid_tgid() >> 32;go_http2_symaddrs_t* symaddrs = bpf_map_lookup_elem(&http2_symaddrs_map, &pid);if (symaddrs == NULL) {log_trace("uprobe__probe_loopy_writer_write_header:writeHeader:symaddrs is NULL\n");return 0;}

 偏移量存储结构体:

typedef struct {// ---- itable symbols ----// io.Writer interface types.__s64 http_http2bufferedWriter;  // "go.itab.*net/http.http2bufferedWriter,io.Writer__s64 transport_bufWriter;  // "google.golang.org/grpc/internal/transport.bufWriter,io.Writer// ---- function argument locations ----// Arguments of net/http.(*http2Framer).WriteDataPadded.location_t http2Framer_WriteDataPadded_f_loc;          // 8location_t http2Framer_WriteDataPadded_streamID_loc;   // 16location_t http2Framer_WriteDataPadded_endStream_loc;  // 20location_t http2Framer_WriteDataPadded_data_ptr_loc;   // 24location_t http2Framer_WriteDataPadded_data_len_loc;   // 32// Arguments of golang.org/x/net/http2.(*Framer).WriteDataPadded.location_t http2_WriteDataPadded_f_loc;          // 8location_t http2_WriteDataPadded_streamID_loc;   // 16location_t http2_WriteDataPadded_endStream_loc;  // 20location_t http2_WriteDataPadded_data_ptr_loc;   // 24location_t http2_WriteDataPadded_data_len_loc;   // 32// Arguments of net/http.(*http2Framer).checkFrameOrder.location_t http2Framer_checkFrameOrder_fr_loc;  // 8location_t http2Framer_checkFrameOrder_f_loc;   // 16// Arguments of golang.org/x/net/http2.(*Framer).checkFrameOrder.location_t http2_checkFrameOrder_fr_loc;  // 8location_t http2_checkFrameOrder_f_loc;   // 16// Arguments of net/http.(*http2writeResHeaders).writeFrame.location_t writeFrame_w_loc;    // 8location_t writeFrame_ctx_loc;  // 16// Arguments of golang.org/x/net/http2/hpack.(*Encoder).WriteField.location_t WriteField_e_loc;  // 8// Note that the HeaderField `f` is further broken down to its name and value members.// This is done so we can better control the location of these members from user-space.// In theory, there could be an ABI that splits these two members across stack and registers.location_t WriteField_f_name_loc;   // 16location_t WriteField_f_value_loc;  // 32// Arguments of net/http.(*http2serverConn).processHeaders.location_t processHeaders_sc_loc;  // 8location_t processHeaders_f_loc;   // 16// Arguments of google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders.location_t http2Server_operateHeaders_t_loc;      // 8location_t http2Server_operateHeaders_frame_loc;  // 16// Arguments of google.golang.org/grpc/internal/transport.(*http2Client).operateHeaders.location_t http2Client_operateHeaders_t_loc;      // 8location_t http2Client_operateHeaders_frame_loc;  // 16// Arguments of google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader.location_t writeHeader_l_loc;          // 8location_t writeHeader_streamID_loc;   // 16location_t writeHeader_endStream_loc;  // 20slice_location_t writeHeader_hf_loc;     // 24// ---- struct member offsets ----// Struct member offsets.// Naming maintains golang style: __offset// Note: values in comments represent known offsets, in case we need to fall back.//       Eventually, they should be removed, because they are not reliable.// Members of golang.org/x/net/http2/hpack.HeaderField.int32_t HeaderField_Name_offset;   // 0int32_t HeaderField_Value_offset;  // 16// Members of google.golang.org/grpc/internal/transport.http2Server.int32_t http2Server_conn_offset;  // 16 or 24// Members of google.golang.org/grpc/internal/transport.http2Client.int32_t http2Client_conn_offset;  // 64// Members of google.golang.org/grpc/internal/transport.loopyWriter.int32_t loopyWriter_framer_offset;  // 40// Members of golang.org/x/net/net/http2.Framer.int32_t Framer_w_offset;  // 112// Members of golang.org/x/net/http2.MetaHeadersFrame.int32_t MetaHeadersFrame_HeadersFrame_offset;  // 0int32_t MetaHeadersFrame_Fields_offset;        // 0// Members of golang.org/x/net/http2.HeadersFrame.int32_t HeadersFrame_FrameHeader_offset;  // 0// Members of golang.org/x/net/http2.FrameHeader.int32_t FrameHeader_Type_offset;      // 1int32_t FrameHeader_Flags_offset;     // 2int32_t FrameHeader_StreamID_offset;  // 8// Members of golang.org/x/net/http2.DataFrame.int32_t DataFrame_data_offset;  // 16// Members of google.golang.org/grpc/internal/transport.bufWriter.int32_t bufWriter_conn_offset;  // 40// Members of net/http.http2serverConn.int32_t http2serverConn_conn_offset;          // 16int32_t http2serverConn_hpackEncoder_offset;  // 360// Members of net/http.http2HeadersFrameint32_t http2HeadersFrame_http2FrameHeader_offset;  // 0// Members of net/http.http2FrameHeader.int32_t http2FrameHeader_Type_offset;      // 1int32_t http2FrameHeader_Flags_offset;     // 2int32_t http2FrameHeader_StreamID_offset;  // 8// Members of golang.org/x/net/http2.DataFrame.int32_t http2DataFrame_data_offset;  // 16// Members of net/http.http2writeResHeaders.int32_t http2writeResHeaders_streamID_offset;   // 0int32_t http2writeResHeaders_endStream_offset;  // 48// Members of net/http.http2MetaHeadersFrame.int32_t http2MetaHeadersFrame_http2HeadersFrame_offset;  // 0int32_t http2MetaHeadersFrame_Fields_offset;             // 8// Members of net/http.http2Framer.int32_t http2Framer_w_offset;  // 112// Members of net/http.http2bufferedWriterint32_t http2bufferedWriter_w_offset;  // 0
} go_http2_symaddrs_t;

读取偏移量:

  void* loopy_writer_ptr = NULL;if (read_location(ctx, &symaddrs->writeHeader_l_loc,sizeof(loopy_writer_ptr), &loopy_writer_ptr)) {log_trace("uprobe__probe_loopy_writer_write_header:5\n");return 0;}

六、如何记录错误调试

1、打日志调试

#define log_trace(fmt, ...)                                        \({                                                             \char ____fmt[] = fmt;                                      \bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \})/* Macro to output debug logs to /sys/kernel/debug/tracing/trace_pipe*/
#if DEBUG == 1
#define log_debug(fmt, ...)                                        \({                                                             \char ____fmt[] = fmt;                                      \bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \})
#else

使用:

log_debug("[grpc-c:lookup_version:]grpc-c version is not support;pid:%d;version:%d;\n", pid, *version);

2、记录metrics

主要记录行号和文件

#ifndef TRACER_TELEMETRY
#define TRACER_TELEMETRY#ifndef TRACER_TELEMETRY_KEY_LIMIT
#define TRACER_TELEMETRY_KEY_LIMIT 4096
#endiftypedef struct {char file[TRACER_TELEMETRY_KEY_LIMIT];uint64_t line;
} tracer_telemetry_key;BPF_HASH_MAP(tracer_telemetry, tracer_telemetry_key, uint64_t, 1024);
BPF_PERCPU_ARRAY_MAP(tracer_telemetry_heap, __u32, tracer_telemetry_key, 1);static __always_inline tracer_telemetry_key* alloc_tracer_telemetry_key() {uint32_t kZero = 0;tracer_telemetry_key* value = bpf_map_lookup_elem(&tracer_telemetry_heap, &kZero);if (value == NULL) {return NULL;}value->line = 0;return value;
}static __always_inline void increment_tracer_telemetry_count(const char* file, uint64_t line) {tracer_telemetry_key* key = alloc_tracer_telemetry_key();if (key == NULL) {return;}if (bpf_probe_read_str(&key->file, sizeof(key->file), file) == -1) {return;}key->line = line;uint64_t *val = NULL;val = bpf_map_lookup_elem(&tracer_telemetry, key);if (val == NULL) {log_debug("tracer:key->file:%s;file:%s\n", key->file, file);uint64_t tmp_value = 0;bpf_map_update_with_telemetry(tracer_telemetry, key, &tmp_value, BPF_NOEXIST);return;}val = bpf_map_lookup_elem(&tracer_telemetry, key);if (val == NULL) {return;}__sync_fetch_and_add(val, 1);
}#ifndef INCR_TRACER_COUNT
#define  INCR_TRACER_COUNT  increment_tracer_telemetry_count(__FILE__, __LINE__)
#endif#endif