From 87da3e3437ca8318a9dbd0dbea4eba87bd69115b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 19 Jan 2015 16:10:54 +0200 Subject: [PATCH] crit: add --format hex option Pavel reported that decimal values for some fields are hard to read, because people used to see hex values in there. Unfortunately, json doesn't support hex representation of integers, so we can only store them as hex strings. Not all field need to be represented as hex strings, so this set introduces a custom field option called "criu" to use in our proto files. One should use [(criu).hex = true] to mark which field should be represented as a hex string. pb2dict module from pycriu package will look into field options and if he finds that criu.hex is set to True, it will convert such field to/from hex string. Though, such behaviour is optional and user can request it by specifying --format hex when calling crit decode("crit encode" in its turn, detects such fields automatically and doesn't require any special cmdline options to be set). We need our proto files to compile with both protoc and protoc-c compilers, which requires creating google/protobuf directory with a symlink to /usr/include/google/protobuf/ descriptor.proto to make protoc-c and generated c files happy. Reported-by: Pavel Emelyanov Signed-off-by: Ruslan Kuprieiev Signed-off-by: Pavel Emelyanov --- .gitignore | 2 ++ crit | 14 +++++---- protobuf/Makefile | 6 ++++ protobuf/google/protobuf/descriptor.proto | 1 + pycriu/images/Makefile | 2 +- pycriu/images/images.py | 24 ++++++++-------- pycriu/images/pb2dict.py | 35 ++++++++++++++++++----- 7 files changed, 59 insertions(+), 25 deletions(-) create mode 120000 protobuf/google/protobuf/descriptor.proto diff --git a/.gitignore b/.gitignore index 977c782aa..3f284972a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ include/syscall.h include/syscall-codes.h protobuf/*.c protobuf/*.h +protobuf/google/protobuf/*.c +protobuf/google/protobuf/*.h include/version.h arch/x86/sys-exec-tbl.c arch/x86/syscalls.S diff --git a/crit b/crit index e5d9b0871..0c7f5e97d 100755 --- a/crit +++ b/crit @@ -7,7 +7,7 @@ import pycriu def handle_cmdline_opts(): desc = 'CRiu Image Tool' - parser = argparse.ArgumentParser(description=desc) + parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('command', choices = ['decode', 'encode'], help = 'decode/encode - convert criu image from/to binary type to/from json') @@ -19,8 +19,12 @@ def handle_cmdline_opts(): help = 'output file (stdout by default)') parser.add_argument('-f', '--format', - choices = ['raw', 'nice'], - help = 'well-formated output (by default: raw for files and nice for stdout)') + choices = ['raw', 'nice', 'hex'], + nargs = '+', + default = [], + help = 'raw - all in one line\n'\ + 'nice - add indents and newlines to look nice(default for stdout)\n'\ + 'hex - print int fields as hex strings where suitable(could be combined with others)') opts = vars(parser.parse_args()) @@ -41,10 +45,10 @@ def outf(opts): def decode(opts): indent = None - img = pycriu.images.load(inf(opts)) + img = pycriu.images.load(inf(opts), opts['format']) # For stdout --format nice is set by default. - if opts['format'] == 'nice' or (opts['format'] == None and opts['out'] == None): + if 'nice' in opts['format'] or ('raw' not in opts['format'] and opts['out'] == None): indent = 4 f = outf(opts) diff --git a/protobuf/Makefile b/protobuf/Makefile index 983629e13..0c3196b3c 100644 --- a/protobuf/Makefile +++ b/protobuf/Makefile @@ -63,6 +63,8 @@ proto-obj-y += rpc.o proto-obj-y += ext-file.o proto-obj-y += cgroup.o proto-obj-y += userns.o +proto-obj-y += google/protobuf/descriptor.o # To make protoc-c happy and compile opts.proto +proto-pbj-y += opts.o proto := $(proto-obj-y:.o=) proto-c := $(proto-obj-y:.o=.pb-c.c) @@ -124,4 +126,8 @@ ifneq ($(MAKECMDGOALS),clean) endif cleanup-y += $(obj)/*.c.d $(obj)/*.pb-c.c $(obj)/*.pb-c.h +cleanup-y += $(obj)/google/protobuf/*.d +cleanup-y += $(obj)/google/protobuf/*.h +cleanup-y += $(obj)/google/protobuf/*.c +cleanup-y += $(obj)/google/protobuf/*.o cleanup-y += $(obj)/*.d $(obj)/*.i $(obj)/*.s $(obj)/*.o diff --git a/protobuf/google/protobuf/descriptor.proto b/protobuf/google/protobuf/descriptor.proto new file mode 120000 index 000000000..07a4c9add --- /dev/null +++ b/protobuf/google/protobuf/descriptor.proto @@ -0,0 +1 @@ +/usr/include/google/protobuf/descriptor.proto \ No newline at end of file diff --git a/pycriu/images/Makefile b/pycriu/images/Makefile index 6aa115ec6..579b6fafe 100644 --- a/pycriu/images/Makefile +++ b/pycriu/images/Makefile @@ -10,7 +10,7 @@ proto-py-modules := $(foreach m,$(proto),$(subst -,_,$(notdir $(m:.proto=_pb2))) # Unfortunately, we can't drop ugly _pb2 suffixes here, because # some _pb2 files depend on others _pb2 files. protobuf: - $(Q) protoc -I=$(SRC_DIR)/protobuf --python_out=./ $(proto) + $(Q) protoc -I=$(SRC_DIR)/protobuf -I=/usr/include/ --python_out=./ $(proto) magic.py: $(SRC_DIR)/scripts/magic-gen.py $(SRC_DIR)/include/magic.h $(E) " GEN " $@ diff --git a/pycriu/images/images.py b/pycriu/images/images.py index 83a1e3ef7..8c4b73eb9 100644 --- a/pycriu/images/images.py +++ b/pycriu/images/images.py @@ -59,7 +59,7 @@ class entry_handler: self.payload = payload self.extra_handler = extra_handler - def load(self, f): + def load(self, f, fmt = None): """ Convert criu image entries from binary format to dict(json). Takes a file-like object and returnes a list with entries in @@ -77,7 +77,7 @@ class entry_handler: break size, = struct.unpack('i', buf) pb.ParseFromString(f.read(size)) - entry = pb2dict.pb2dict(pb) + entry = pb2dict.pb2dict(pb, fmt) # Read extra if self.extra_handler: @@ -87,12 +87,12 @@ class entry_handler: return entries - def loads(self, s): + def loads(self, s, fmt = None): """ Same as load(), but takes a string as an argument. """ f = io.BytesIO(s) - return self.load(f) + return self.load(f, fmt) def dump(self, entries, f): """ @@ -131,7 +131,7 @@ class pagemap_handler: that it has a header of pagemap_head type followed by entries of pagemap_entry type. """ - def load(self, f): + def load(self, f, fmt = None): entries = [] pb = pagemap_head() @@ -141,15 +141,15 @@ class pagemap_handler: break size, = struct.unpack('i', buf) pb.ParseFromString(f.read(size)) - entries.append(pb2dict.pb2dict(pb)) + entries.append(pb2dict.pb2dict(pb, fmt)) pb = pagemap_entry() return entries - def loads(self, s): + def loads(self, s, fmt = None): f = io.BytesIO(s) - return self.load(f) + return self.load(f, fmt) def dump(self, entries, f): pb = pagemap_head() @@ -258,7 +258,7 @@ handlers = { 'IPCNS_MSG' : entry_handler(ipc_msg_entry) } -def load(f): +def load(f, fmt = None): """ Convert criu image from binary format to dict(json). Takes a file-like object to read criu image from. @@ -281,16 +281,16 @@ def load(f): raise Exception("No handler found for image with such magic "+m) image['magic'] = m - image['entries'] = handler.load(f) + image['entries'] = handler.load(f, fmt) return image -def loads(s): +def loads(s, fmt = None): """ Same as load(), but takes a string. """ f = io.BytesIO(s) - return load(f) + return load(f, fmt) def dump(img, f): """ diff --git a/pycriu/images/pb2dict.py b/pycriu/images/pb2dict.py index 815870b16..d8177936e 100644 --- a/pycriu/images/pb2dict.py +++ b/pycriu/images/pb2dict.py @@ -1,4 +1,5 @@ from google.protobuf.descriptor import FieldDescriptor as FD +import opts_pb2 # pb2dict and dict2pb are methods to convert pb to/from dict. # Inspired by: @@ -39,19 +40,33 @@ _basic_cast = { FD.TYPE_STRING : unicode } -def _pb2dict_cast(field, value): +def _marked_as_hex(field): + return field.GetOptions().Extensions[opts_pb2.criu].hex + +def _pb2dict_cast(field, value, fmt = None, is_hex = False): + if not is_hex: + is_hex = _marked_as_hex(field) + if fmt: + print_hex = 'hex' in fmt + if field.type == FD.TYPE_MESSAGE: - return pb2dict(value) + return pb2dict(value, fmt, is_hex) elif field.type == FD.TYPE_BYTES: return value.encode('base64') elif field.type == FD.TYPE_ENUM: return field.enum_type.values_by_number.get(value, None).name elif field.type in _basic_cast: - return _basic_cast[field.type](value) + cast = _basic_cast[field.type] + if (cast == int or cast == long) and is_hex and print_hex: + # Fields that have (criu).hex = true option set + # should be stored in hex string format. + return "0x%x" % value + else: + return cast(value) else: raise Exception("Field(%s) has unsupported type %d" % (field.name, field.type)) -def pb2dict(pb): +def pb2dict(pb, fmt = None, is_hex = False): """ Convert protobuf msg to dictionary. Takes a protobuf message and returns a dict. @@ -61,9 +76,9 @@ def pb2dict(pb): if field.label == FD.LABEL_REPEATED: d_val = [] for v in value: - d_val.append(_pb2dict_cast(field, v)) + d_val.append(_pb2dict_cast(field, v, fmt, is_hex)) else: - d_val = _pb2dict_cast(field, value) + d_val = _pb2dict_cast(field, value, fmt, is_hex) d[field.name] = d_val return d @@ -77,7 +92,13 @@ def _dict2pb_cast(field, value): elif field.type == FD.TYPE_ENUM: return field.enum_type.values_by_name.get(value, None).number else: - return _basic_cast[field.type](value) + cast = _basic_cast[field.type] + if (cast == int or cast == long) and isinstance(value, unicode): + # Some int or long fields might be stored as hex + # strings. See _pb2dict_cast. + return cast(value, 0) + else: + return cast(value) def dict2pb(d, pb): """