crit: add --format hex option

Pavel reported that decimal values for some fields are hard to read,
because people used to see hex values in there. Unfortunately, json
doesn't support hex representation of integers, so we can only store
them as hex strings. Not all field need to be represented as hex
strings, so this set introduces a custom field option called "criu"
to use in our proto files. One should use [(criu).hex = true] to mark
which field should be represented as a hex string. pb2dict module
from pycriu package will look into field options and if he finds that
criu.hex is set to True, it will convert such field to/from hex string.
Though, such behaviour is optional and user can request it by specifying
 --format hex when calling crit decode("crit encode" in its turn, detects
such fields automatically and doesn't require any special cmdline options
to be set).

We need our proto files to compile with both protoc and
protoc-c compilers, which requires creating google/protobuf
directory with a symlink to /usr/include/google/protobuf/
descriptor.proto to make protoc-c and generated c files happy.

Reported-by: Pavel Emelyanov <xemul@parallels.com>
Signed-off-by: Ruslan Kuprieiev <kupruser@gmail.com>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

.gitignore

@@ -20,6 +20,8 @@ include/syscall.h
 include/syscall-codes.h
 protobuf/*.c
 protobuf/*.h
+protobuf/google/protobuf/*.c
+protobuf/google/protobuf/*.h
 include/version.h
 arch/x86/sys-exec-tbl.c
 arch/x86/syscalls.S

crit

@@ -7,7 +7,7 @@ import pycriu
 
 def handle_cmdline_opts():
 	desc = 'CRiu Image Tool'
-	parser = argparse.ArgumentParser(description=desc)
+	parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter)
 	parser.add_argument('command',
 			    choices = ['decode', 'encode'],
 			help = 'decode/encode - convert criu image from/to binary type to/from json')
@@ -19,8 +19,12 @@ def handle_cmdline_opts():
 			help = 'output file (stdout by default)')
 	parser.add_argument('-f',
 			    '--format',
-			    choices = ['raw', 'nice'],
-			help = 'well-formated output (by default: raw for files and nice for stdout)')
+			    choices = ['raw', 'nice', 'hex'],
+			    nargs = '+',
+			    default = [],
+			help = 'raw  - all in one line\n'\
+			       'nice - add indents and newlines to look nice(default for stdout)\n'\
+			       'hex  - print int fields as hex strings where suitable(could be combined with others)')
 
 	opts = vars(parser.parse_args())
 
@@ -41,10 +45,10 @@ def outf(opts):
 
 def decode(opts):
 	indent = None
-	img = pycriu.images.load(inf(opts))
+	img = pycriu.images.load(inf(opts), opts['format'])
 
 	# For stdout --format nice is set by default.
-	if opts['format'] == 'nice' or (opts['format'] == None and opts['out'] == None):
+	if 'nice' in opts['format'] or ('raw' not in opts['format'] and opts['out'] == None):
 		indent = 4
 
 	f = outf(opts)

protobuf/Makefile

@@ -63,6 +63,8 @@ proto-obj-y	+= rpc.o
 proto-obj-y	+= ext-file.o
 proto-obj-y	+= cgroup.o
 proto-obj-y	+= userns.o
+proto-obj-y	+= google/protobuf/descriptor.o # To make protoc-c happy and compile opts.proto
+proto-pbj-y	+= opts.o
 
 proto		:= $(proto-obj-y:.o=)
 proto-c		:= $(proto-obj-y:.o=.pb-c.c)
@@ -124,4 +126,8 @@ ifneq ($(MAKECMDGOALS),clean)
 endif
 
 cleanup-y += $(obj)/*.c.d $(obj)/*.pb-c.c $(obj)/*.pb-c.h
+cleanup-y += $(obj)/google/protobuf/*.d
+cleanup-y += $(obj)/google/protobuf/*.h
+cleanup-y += $(obj)/google/protobuf/*.c
+cleanup-y += $(obj)/google/protobuf/*.o
 cleanup-y += $(obj)/*.d $(obj)/*.i $(obj)/*.s $(obj)/*.o

protobuf/google/protobuf/descriptor.proto

@@ -0,0 +1 @@
+/usr/include/google/protobuf/descriptor.proto

pycriu/images/Makefile

@@ -10,7 +10,7 @@ proto-py-modules := $(foreach m,$(proto),$(subst -,_,$(notdir $(m:.proto=_pb2)))
 # Unfortunately, we can't drop ugly _pb2 suffixes here, because
 # some _pb2 files depend on others _pb2 files.
 protobuf:
-	$(Q) protoc -I=$(SRC_DIR)/protobuf --python_out=./ $(proto)
+	$(Q) protoc -I=$(SRC_DIR)/protobuf -I=/usr/include/ --python_out=./ $(proto)
 
 magic.py: $(SRC_DIR)/scripts/magic-gen.py $(SRC_DIR)/include/magic.h
 	$(E) "  GEN  " $@

pycriu/images/images.py

@@ -59,7 +59,7 @@ class entry_handler:
 		self.payload		= payload
 		self.extra_handler	= extra_handler
 
-	def load(self, f):
+	def load(self, f, fmt = None):
 		"""
 		Convert criu image entries from binary format to dict(json).
 		Takes a file-like object and returnes a list with entries in
@@ -77,7 +77,7 @@ class entry_handler:
 				break
 			size, = struct.unpack('i', buf)
 			pb.ParseFromString(f.read(size))
-			entry = pb2dict.pb2dict(pb)
+			entry = pb2dict.pb2dict(pb, fmt)
 
 			# Read extra
 			if self.extra_handler:
@@ -87,12 +87,12 @@ class entry_handler:
 
 		return entries
 
-	def loads(self, s):
+	def loads(self, s, fmt = None):
 		"""
 		Same as load(), but takes a string as an argument.
 		"""
 		f = io.BytesIO(s)
-		return self.load(f)
+		return self.load(f, fmt)
 
 	def dump(self, entries, f):
 		"""
@@ -131,7 +131,7 @@ class pagemap_handler:
 	that it has a header of pagemap_head type followed by entries
 	of pagemap_entry type.
 	"""
-	def load(self, f):
+	def load(self, f, fmt = None):
 		entries = []
 
 		pb = pagemap_head()
@@ -141,15 +141,15 @@ class pagemap_handler:
 				break
 			size, = struct.unpack('i', buf)
 			pb.ParseFromString(f.read(size))
-			entries.append(pb2dict.pb2dict(pb))
+			entries.append(pb2dict.pb2dict(pb, fmt))
 
 			pb = pagemap_entry()
 
 		return entries
 
-	def loads(self, s):
+	def loads(self, s, fmt = None):
 		f = io.BytesIO(s)
-		return self.load(f)
+		return self.load(f, fmt)
 
 	def dump(self, entries, f):
 		pb = pagemap_head()
@@ -258,7 +258,7 @@ handlers = {
 	'IPCNS_MSG'		: entry_handler(ipc_msg_entry)
 	}
 
-def load(f):
+def load(f, fmt = None):
 	"""
 	Convert criu image from binary format to dict(json).
 	Takes a file-like object to read criu image from.
@@ -281,16 +281,16 @@ def load(f):
 		raise Exception("No handler found for image with such magic "+m)
 
 	image['magic']		= m
-	image['entries']	= handler.load(f)
+	image['entries']	= handler.load(f, fmt)
 
 	return image
 
-def loads(s):
+def loads(s, fmt = None):
 	"""
 	Same as load(), but takes a string.
 	"""
 	f = io.BytesIO(s)
-	return load(f)
+	return load(f, fmt)
 
 def dump(img, f):
 	"""

pycriu/images/pb2dict.py

@@ -1,4 +1,5 @@
 from google.protobuf.descriptor import FieldDescriptor as FD
+import opts_pb2
 
 # pb2dict and dict2pb are methods to convert pb to/from dict.
 # Inspired by:
@@ -39,19 +40,33 @@ _basic_cast = {
 	FD.TYPE_STRING		: unicode
 }
 
-def _pb2dict_cast(field, value):
+def _marked_as_hex(field):
+	return field.GetOptions().Extensions[opts_pb2.criu].hex
+
+def _pb2dict_cast(field, value, fmt = None, is_hex = False):
+	if not is_hex:
+		is_hex = _marked_as_hex(field)
+	if fmt:
+		print_hex = 'hex' in fmt
+
 	if field.type == FD.TYPE_MESSAGE:
-		return pb2dict(value)
+		return pb2dict(value, fmt, is_hex)
 	elif field.type == FD.TYPE_BYTES:
 		return value.encode('base64')
 	elif field.type == FD.TYPE_ENUM:
 		return field.enum_type.values_by_number.get(value, None).name
 	elif field.type in _basic_cast:
-		return _basic_cast[field.type](value)
+		cast = _basic_cast[field.type]
+		if (cast == int or cast == long) and is_hex and print_hex:
+			# Fields that have (criu).hex = true option set
+			# should be stored in hex string format.
+			return "0x%x" % value
+		else:
+			return cast(value)
 	else:
 		raise Exception("Field(%s) has unsupported type %d" % (field.name, field.type))
 
-def pb2dict(pb):
+def pb2dict(pb, fmt = None, is_hex = False):
 	"""
 	Convert protobuf msg to dictionary.
 	Takes a protobuf message and returns a dict.
@@ -61,9 +76,9 @@ def pb2dict(pb):
 		if field.label == FD.LABEL_REPEATED:
 			d_val = []
 			for v in value:
-				d_val.append(_pb2dict_cast(field, v))
+				d_val.append(_pb2dict_cast(field, v, fmt, is_hex))
 		else:
-			d_val = _pb2dict_cast(field, value)
+			d_val = _pb2dict_cast(field, value, fmt, is_hex)
 
 		d[field.name] = d_val
 	return d
@@ -77,7 +92,13 @@ def _dict2pb_cast(field, value):
 	elif field.type == FD.TYPE_ENUM:
 		return field.enum_type.values_by_name.get(value, None).number
 	else:
-		return _basic_cast[field.type](value)
+		cast = _basic_cast[field.type]
+		if (cast == int or cast == long) and isinstance(value, unicode):
+			# Some int or long fields might be stored as hex
+			# strings. See _pb2dict_cast.
+			return cast(value, 0)
+		else:
+			return cast(value)
 
 def dict2pb(d, pb):
 	"""