WIP: Initial RTrace refactoring

* Plan is to create util/trace.c with unified api
  to store debug and esil traces all together but
  keeping proper consistency to be used for sessions,
  stepback, ssa analysis for graphs and more

libr/anal/anal.c

@@ -480,7 +480,6 @@ R_API void r_anal_purge(RAnal *anal) {
  r_anal_purge_imports (anal);
 }
 
-
 static int default_archinfo(int res, int q) {
  if (res < 1) {
  return 1;
@@ -587,7 +586,7 @@ R_API bool r_anal_noreturn_add(RAnal *anal, const char *name, ut64 addr) {
  return true;
  }
  }
- if (name && *name) {
+ if (R_STR_ISNOTEMPTY (name)) {
  tmp_name = name;
  } else {
  RAnalFunction *fcn = r_anal_get_fcn_in (anal, addr, -1);

libr/core/cmd_debug.c

@@ -491,7 +491,7 @@ static RCoreHelpMessage help_msg_dt = {
  "dte", "[?]", "show esil trace logs",
  "dtg", "", "graph call/ret trace",
  "dtg*", "", "graph in agn/age commands. use .dtg*;aggi for visual",
- "dtgi", "", "interactive debug trace",
+ "dtgv", "", "visual / interactive debug trace graph view",
  "dts", "[?]", "trace sessions",
  "dtt", " [tag]", "select trace tag (no arg unsets)",
  NULL
@@ -514,7 +514,7 @@ static RCoreHelpMessage help_msg_dts = {
  "dts-", "", "stop trace session",
  "dtst", " [dir] ", "save trace sessions to disk",
  "dtsf", " [dir] ", "read trace sessions from disk",
- "dtsm", "", "list current memory map and hash",
+ "dtsl", "", "list current memory map and hash",
  NULL
 };
 
@@ -5605,13 +5605,15 @@ static int cmd_debug(void *data, const char *input) {
  core->dbg->session = r_debug_session_new ();
  r_debug_session_load (core->dbg, input + 4);
  break;
- case 'm': // "dtsm"
+ case 'l': // "dtsl"
+ case 'm': // "dtsm" // DEPRECATE dstm command, dtsl is a better name
  if (core->dbg->session) {
  r_debug_session_list_memory (core->dbg);
  }
  break;
  default:
  r_core_cmd_help (core, help_msg_dts);
+ break;
  }
  break;
  case '?':

libr/core/disasm.c

@@ -113,7 +113,7 @@ typedef struct r_disasm_state_t {
  bool show_lines_call;
  bool show_lines_fcn;
  bool linesright;
- int tracespace;
+ bool tracespace;
  int cyclespace;
  int cmtfold;
  int show_indent;
@@ -719,7 +719,7 @@ static RDisasmState *ds_init(RCore *core) {
  ds->linesright = r_config_get_i (core->config, "asm.lines.right");
  ds->show_indent = r_config_get_i (core->config, "asm.indent");
  ds->indent_space = r_config_get_i (core->config, "asm.indentspace");
- ds->tracespace = r_config_get_i (core->config, "asm.trace.space");
+ ds->tracespace = r_config_get_b (core->config, "asm.trace.space");
  ds->cyclespace = r_config_get_i (core->config, "asm.cyclespace");
  ds->show_dwarf = r_config_get_i (core->config, "asm.dwarf");
  ds->dwarfFile = r_config_get_i (ds->core->config, "asm.dwarf.file");

libr/debug/dsession.c

@@ -454,16 +454,11 @@ R_API bool r_debug_session_save(RDebugSession *session, const char *path) {
  return false;
  }
  r_debug_session_serialize (session, db);
-
- if (!session_sdb_save (db, path)) {
- sdb_free (db);
- return false;
- }
+ bool res = session_sdb_save (db, path);
  sdb_free (db);
- return true;
+ return res;
 }
 
-
 #define CHECK_TYPE(v,t) \
  if (!v || v->type != t) \
  continue

libr/debug/trace.c

@@ -40,6 +40,7 @@ R_API int r_debug_trace_tag(RDebug *dbg, int tag) {
  return ntag;
 }
 
+// this function belongs to dsession
 R_API bool r_debug_trace_ins_before(RDebug *dbg) {
  RListIter *it, *it_tmp;
  RAnalValue *val;
@@ -104,6 +105,7 @@ R_API bool r_debug_trace_ins_before(RDebug *dbg) {
  return true;
 }
 
+// this function belongs to dsession.c instead of trace.c
 R_API bool r_debug_trace_ins_after(RDebug *dbg) {
  RListIter *it;
  RAnalValue *val;
@@ -159,6 +161,7 @@ R_API bool r_debug_trace_ins_after(RDebug *dbg) {
 /*
  * something happened at the given pc that we need to trace
  */
+// this belongs to dsession.c
 R_API bool r_debug_trace_pc(RDebug *dbg, ut64 pc) {
  r_return_val_if_fail (dbg && dbg->trace, false);
  ut8 buf[32];
@@ -177,6 +180,7 @@ R_API bool r_debug_trace_pc(RDebug *dbg, ut64 pc) {
  return true;
 }
 
+// belongs to dsession
 R_API void r_debug_trace_op(RDebug *dbg, RAnalOp *op) {
  r_return_if_fail (dbg && dbg->trace);
  static ut64 oldpc = UT64_MAX; // Must trace the previously traced instruction

libr/esil/esil_trace.c

@@ -9,7 +9,7 @@
 #define CMP_REG_CHANGE(x, y) ((x) - ((REsilRegChange *)y)->idx)
 #define CMP_MEM_CHANGE(x, y) ((x) - ((REsilMemChange *)y)->idx)
 
-static R_TH_LOCAL int ocbs_set = false;
+static R_TH_LOCAL bool ocbs_set = false;
 static R_TH_LOCAL REsilCallbacks ocbs = {0};
 
 static void htup_vector_free(HtUPKv *kv) {
@@ -57,7 +57,7 @@ R_API REsilTrace *r_esil_trace_new(REsil *esil) {
  if (!b) {
  goto error;
  }
- if (b->bytes && a->bytes && b->size > 0) {
+ if (a->bytes && b->bytes && b->size > 0) {
  memcpy (b->bytes, a->bytes, b->size);
  }
  trace->arena[i] = b;
@@ -81,7 +81,7 @@ R_API void r_esil_trace_free(REsilTrace *trace) {
  // eprintf ("sdb free %p%c", trace->db, 10);
  sdb_free (trace->db);
  trace->db = NULL;
- R_FREE (trace);
+ free (trace);
  }
 }
 
@@ -247,6 +247,7 @@ R_API void r_esil_trace_op(REsil *esil, RAnalOp *op) {
  //eprintf ("[ESIL] OPCODE %s\n", op->mnemonic);
  //eprintf ("[ESIL] EXPR = %s\n", expr);
  /* set hooks */
+ // TODO: esil->cb.user = &globals;
  esil->cb.hook_reg_read = trace_hook_reg_read;
  esil->cb.hook_reg_write = trace_hook_reg_write;
  esil->cb.hook_mem_read = trace_hook_mem_read;

libr/include/r_debug.h

@@ -224,34 +224,6 @@ typedef struct r_snap_entry {
  int perm;
 } RSnapEntry;
 
-typedef struct r_debug_trace_t {
- RList *traces;
- int count;
- int enabled;
- int tag;
- int dup;
- char *addresses;
- HtPP *ht; // use rbtree like the iocache?
-} RDebugTrace;
-
-// R2_590 rename to traceitem for consistency?
-#define r_debug_tracepoint_free(x) free((x))
-typedef struct r_debug_tracepoint_t {
- ut64 addr;
- ut64 tags; // XXX
- int tag; // XXX
- int size;
- int count;
- int times;
- ut64 stamp;
-#if 0
- // registers accessed
- // memory access
- ut64 refaddr;
- int direction
-#endif
-} RDebugTracepoint;
-
 typedef struct r_debug_t {
  // R2_590 use RArchConfig instead
  char *arch;

libr/util/trace.c

@@ -0,0 +1,70 @@
+/* radare - LGPL - Copyright 2023 - pancake */
+
+typedef struct {
+ ut64 addr;
+ bool write;
+ const char *reg;
+ ut64 value;
+ int size;
+} RTraceAccessRegister;
+
+typedef struct {
+ bool write;
+ ut64 addr;
+ const ut8 *buf;
+ size_t buflen;
+} RTraceAccessMemory;
+
+typedef struct {
+ bool ismem;
+ union {
+ RDebugTraceAccessMemory mem;
+ RDebugTraceAccessRegister reg;
+ }
+} RTraceAccess;
+
+typedef struct r_trace_t {
+ RList *traces; // can be an RVec
+ int count;
+ int enabled;
+ int tag;
+ int dup;
+ char *addresses;
+ HtPP *ht; // use rbtree like the iocache?
+} RTrace;
+
+
+// R2_590 rename to traceitem for consistency?
+#define r_debug_tracepoint_free(x) free((x))
+typedef struct r_trace_item_t {
+ ut64 addr;
+ ut64 tags; // XXX
+ int tag; // XXX
+ int size;
+ int count; // rename to index
+ int times; // rename to count :jiji:
+ ut64 stamp;
+ RVecAccess access;
+#if 0
+
+ ut64 refaddr;
+ int direction
+#endif
+} RTraceItem;
+
+R_GENERATE_VEC_IMPL_FOR(Access, RTraceAccess);
+
+#if 0
+RVecTraces v;
+RVecTraces_init (&v);
+RDebugTraceAccess t = {
+ .ismem = true,
+ .mem = {
+ .write = true,
+ .addr = 0x80000,
+ .buf = "hell",
+ .buflen = 4,
+ }
+};
+RVecTraces_push_back (&v &t);
+#endif