From b1dc7fae2a95804948ba9eedca08d208cdd5f825 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 26 May 2010 12:19:58 +0200 Subject: [PATCH 001/147] Initial import of spice protocol description and demarshall generator The "spice.proto" file describes in detail the networking prototcol that spice uses and spice_codegen.py can parse this and generate demarshallers for such network messages. --- python_modules/__init__.py | 0 python_modules/codegen.py | 354 +++++++++++ python_modules/demarshal.py | 1033 ++++++++++++++++++++++++++++++ python_modules/ptypes.py | 965 ++++++++++++++++++++++++++++ python_modules/spice_parser.py | 157 +++++ spice.proto | 1086 ++++++++++++++++++++++++++++++++ 6 files changed, 3595 insertions(+) create mode 100644 python_modules/__init__.py create mode 100644 python_modules/codegen.py create mode 100644 python_modules/demarshal.py create mode 100644 python_modules/ptypes.py create mode 100644 python_modules/spice_parser.py create mode 100644 spice.proto diff --git a/python_modules/__init__.py b/python_modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python_modules/codegen.py b/python_modules/codegen.py new file mode 100644 index 0000000..5bb659a --- /dev/null +++ b/python_modules/codegen.py @@ -0,0 +1,354 @@ +from cStringIO import StringIO + +def camel_to_underscores(s, upper = False): + res = "" + for i in range(len(s)): + c = s[i] + if i > 0 and c.isupper(): + res = res + "_" + if upper: + res = res + c.upper() + else: + res = res + c.lower() + return res + +def underscores_to_camel(s): + res = "" + do_upper = True + for i in range(len(s)): + c = s[i] + if c == "_": + do_upper = True + else: + if do_upper: + res = res + c.upper() + else: + res = res + c + do_upper = False + return res + +proto_prefix = "Temp" + +def set_prefix(prefix): + global proto_prefix + global proto_prefix_upper + global proto_prefix_lower + proto_prefix = prefix + proto_prefix_upper = prefix.upper() + proto_prefix_lower = prefix.lower() + +def prefix_underscore_upper(*args): + s = proto_prefix_upper + for arg in args: + s = s + "_" + arg + return s + +def prefix_underscore_lower(*args): + s = proto_prefix_lower + for arg in args: + s = s + "_" + arg + return s + +def prefix_camel(*args): + s = proto_prefix + for arg in args: + s = s + underscores_to_camel(arg) + return s + +def increment_identifier(idf): + v = idf[-1:] + if v.isdigit(): + return idf[:-1] + str(int(v) + 1) + return idf + "2" + +def sum_array(array): + if len(array) == 0: + return 0 + return " + ".join(array) + +class CodeWriter: + def __init__(self): + self.out = StringIO() + self.contents = [self.out] + self.indentation = 0 + self.at_line_start = True + self.indexes = ["i", "j", "k", "ii", "jj", "kk"] + self.current_index = 0 + self.generated = {} + self.vars = [] + self.has_error_check = False + self.options = {} + self.function_helper_writer = None + + def set_option(self, opt, value = True): + self.options[opt] = value + + def has_option(self, opt): + return self.options.has_key(opt) + + def set_is_generated(self, kind, name): + if not self.generated.has_key(kind): + v = {} + self.generated[kind] = v + else: + v = self.generated[kind] + v[name] = 1 + + def is_generated(self, kind, name): + if not self.generated.has_key(kind): + return False + v = self.generated[kind] + return v.has_key(name) + + def getvalue(self): + strs = map(lambda writer: writer.getvalue(), self.contents) + return "".join(strs) + + def get_subwriter(self): + writer = CodeWriter() + self.contents.append(writer) + self.out = StringIO() + self.contents.append(self.out) + writer.indentation = self.indentation + writer.at_line_start = self.at_line_start + writer.generated = self.generated + writer.options = self.options + + return writer; + + def write(self, s): + # Ensure its a string + s = str(s) + + if len(s) == 0: + return + + if self.at_line_start: + for i in range(self.indentation): + self.out.write(" ") + self.at_line_start = False + self.out.write(s) + return self + + def newline(self): + self.out.write("\n") + self.at_line_start = True + return self + + def writeln(self, s): + self.write(s) + self.newline() + return self + + def label(self, s): + self.indentation = self.indentation - 1 + self.write(s + ":") + self.indentation = self.indentation + 1 + self.newline() + + def statement(self, s): + self.write(s) + self.write(";") + self.newline() + return self + + def assign(self, var, val): + self.write("%s = %s" % (var, val)) + self.write(";") + self.newline() + return self + + def increment(self, var, val): + self.write("%s += %s" % (var, val)) + self.write(";") + self.newline() + return self + + def comment(self, str): + self.write("/* " + str + " */") + return self + + def todo(self, str): + self.comment("TODO: *** %s ***" % str).newline() + return self + + def error_check(self, check, label = "error"): + self.has_error_check = True + with self.block("if (SPICE_UNLIKELY(%s))" % check): + if self.has_option("print_error"): + self.statement('printf("%%s: Caught error - %s", __PRETTY_FUNCTION__)' % check) + if self.has_option("assert_on_error"): + self.statement("assert(0)") + self.statement("goto %s" % label) + + def indent(self): + self.indentation += 4; + + def unindent(self): + self.indentation -= 4; + if self.indentation < 0: + self.indenttation = 0 + + def begin_block(self, prefix= "", comment = ""): + if len(prefix) > 0: + self.write(prefix) + if self.at_line_start: + self.write("{") + else: + self.write(" {") + if len(comment) > 0: + self.write(" ") + self.comment(comment) + self.newline() + self.indent() + + def end_block(self, semicolon=False, newline=True): + self.unindent() + if self.at_line_start: + self.write("}") + else: + self.write(" }") + if semicolon: + self.write(";") + if newline: + self.newline() + + class Block: + def __init__(self, writer, semicolon, newline): + self.writer = writer + self.semicolon = semicolon + self.newline = newline + + def __enter__(self): + return self.writer.get_subwriter() + + def __exit__(self, exc_type, exc_value, traceback): + self.writer.end_block(self.semicolon, self.newline) + + class PartialBlock: + def __init__(self, writer, scope, semicolon, newline): + self.writer = writer + self.scope = scope + self.semicolon = semicolon + self.newline = newline + + def __enter__(self): + return self.scope + + def __exit__(self, exc_type, exc_value, traceback): + self.writer.end_block(self.semicolon, self.newline) + + class NoBlock: + def __init__(self, scope): + self.scope = scope + + def __enter__(self): + return self.scope + + def __exit__(self, exc_type, exc_value, traceback): + pass + + def block(self, prefix= "", comment = "", semicolon=False, newline=True): + self.begin_block(prefix, comment) + return self.Block(self, semicolon, newline) + + def partial_block(self, scope, semicolon=False, newline=True): + return self.PartialBlock(self, scope, semicolon, newline) + + def no_block(self, scope): + return self.NoBlock(scope) + + def optional_block(self, scope): + if scope != None: + return self.NoBlock(scope) + return self.block() + + def for_loop(self, index, limit): + return self.block("for (%s = 0; %s < %s; %s++)" % (index, index, limit, index)) + + def while_loop(self, expr): + return self.block("while (%s)" % (expr)) + + def if_block(self, check, elseif=False, newline=True): + s = "if (%s)" % (check) + if elseif: + s = " else " + s + self.begin_block(s, "") + return self.Block(self, False, newline) + + def variable_defined(self, name): + for n in self.vars: + if n == name: + return True + return False + + def variable_def(self, ctype, *names): + for n in names: + # Strip away initialization + i = n.find("=") + if i != -1: + n = n[0:i] + self.vars.append(n.strip()) + # only add space for non-pointer types + if ctype[-1] == "*": + ctype = ctype[:-1].rstrip() + self.writeln("%s *%s;"%(ctype, ", *".join(names))) + else: + self.writeln("%s %s;"%(ctype, ", ".join(names))) + return self + + def function_helper(self): + if self.function_helper_writer != None: + writer = self.function_helper_writer.get_subwriter() + self.function_helper_writer.newline() + else: + writer = self.get_subwriter() + return writer + + def function(self, name, return_type, args, static = False): + self.has_error_check = False + self.function_helper_writer = self.get_subwriter() + if static: + self.write("static ") + self.write(return_type) + self.write(" %s(%s)"% (name, args)).newline() + self.begin_block() + self.function_variables_writer = self.get_subwriter() + self.function_variables = {} + return self.function_variables_writer + + def macro(self, name, args, define): + self.write("#define %s(%s) %s" % (name, args, define)).newline() + + def add_function_variable(self, ctype, name): + if self.function_variables.has_key(name): + assert(self.function_variables[name] == ctype) + else: + self.function_variables[name] = ctype + self.function_variables_writer.variable_def(ctype, name) + + def pop_index(self): + index = self.indexes[self.current_index] + self.current_index = self.current_index + 1 + self.add_function_variable("uint32_t", index) + return index + + def push_index(self): + self.current_index = self.current_index - 1 + + class Index: + def __init__(self, writer, val): + self.writer = writer + self.val = val + + def __enter__(self): + return self.val + + def __exit__(self, exc_type, exc_value, traceback): + self.writer.push_index() + + def index(self, no_block = False): + if no_block: + return self.no_block(None) + val = self.pop_index() + return self.Index(self, val) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py new file mode 100644 index 0000000..fcd6850 --- /dev/null +++ b/python_modules/demarshal.py @@ -0,0 +1,1033 @@ +import ptypes +import codegen + + +def write_parser_helpers(writer): + if writer.is_generated("helper", "demarshaller"): + return + + writer.set_is_generated("helper", "demarshaller") + + writer = writer.function_helper() + + writer.writeln("#ifdef WORDS_BIGENDIAN") + for size in [8, 16, 32, 64]: + for sign in ["", "u"]: + utype = "uint%d" % (size) + type = "%sint%d" % (sign, size) + swap = "SPICE_BYTESWAP%d" % size + if size == 8: + writer.macro("read_%s" % type, "ptr", "(*((%s_t *)(ptr)))" % type) + else: + writer.macro("read_%s" % type, "ptr", "((%s_t)%s(*((%s_t *)(ptr)))" % (type, swap, utype)) + writer.writeln("#else") + for size in [8, 16, 32, 64]: + for sign in ["", "u"]: + type = "%sint%d" % (sign, size) + writer.macro("read_%s" % type, "ptr", "(*((%s_t *)(ptr)))" % type) + writer.writeln("#endif") + + for size in [8, 16, 32, 64]: + for sign in ["", "u"]: + writer.newline() + type = "%sint%d" % (sign, size) + ctype = "%s_t" % type + scope = writer.function("SPICE_GNUC_UNUSED consume_%s" % type, ctype, "uint8_t **ptr", True) + scope.variable_def(ctype, "val") + writer.assign("val", "read_%s(*ptr)" % type) + writer.increment("*ptr", size / 8) + writer.statement("return val") + writer.end_block() + + writer.newline() + writer.statement("typedef struct PointerInfo PointerInfo") + writer.statement("typedef uint8_t * (*parse_func_t)(uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *ptr_info, int minor)") + writer.statement("typedef uint8_t * (*parse_msg_func_t)(uint8_t *message_start, uint8_t *message_end, int minor, size_t *size_out)") + writer.statement("typedef uint8_t * (*spice_parse_channel_func_t)(uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out)") + + writer.newline() + writer.begin_block("struct PointerInfo") + writer.variable_def("uint64_t", "offset") + writer.variable_def("parse_func_t", "parse") + writer.variable_def("SPICE_ADDRESS *", "dest") + writer.variable_def("uint32_t", "nelements") + writer.end_block(semicolon=True) + +def write_read_primitive(writer, start, container, name, scope): + m = container.lookup_member(name) + assert(m.is_primitive()) + writer.assign("pos", start + " + " + container.get_nw_offset(m, "", "__nw_size")) + writer.error_check("pos + %s > message_end" % m.member_type.get_fixed_nw_size()) + + var = "%s__value" % (name) + scope.variable_def(m.member_type.c_type(), var) + writer.assign(var, "read_%s(pos)" % (m.member_type.primitive_type())) + return var + +def write_read_primitive_item(writer, item, scope): + assert(item.type.is_primitive()) + writer.assign("pos", item.get_position()) + writer.error_check("pos + %s > message_end" % item.type.get_fixed_nw_size()) + var = "%s__value" % (item.subprefix) + scope.variable_def(item.type.c_type(), var) + writer.assign(var, "read_%s(pos)" % (item.type.primitive_type())) + return var + +class ItemInfo: + def __init__(self, type, prefix, position): + self.type = type + self.prefix = prefix + self.subprefix = prefix + self.position = position + self.non_null = False + self.member = None + + def nw_size(self): + return self.prefix + "__nw_size" + + def mem_size(self): + return self.prefix + "__mem_size" + + def extra_size(self): + return self.prefix + "__extra_size" + + def get_position(self): + return self.position + +class MemberItemInfo(ItemInfo): + def __init__(self, member, container, start): + if not member.is_switch(): + self.type = member.member_type + self.prefix = member.name + self.subprefix = member.name + self.non_null = member.has_attr("nonnull") + self.position = "(%s + %s)" % (start, container.get_nw_offset(member, "", "__nw_size")) + self.member = member + +def write_validate_switch_member(writer, container, switch_member, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + var = container.lookup_member(switch_member.variable) + var_type = var.member_type + + v = write_read_primitive(writer, start, container, switch_member.variable, parent_scope) + + item = MemberItemInfo(switch_member, container, start) + + first = True + for c in switch_member.cases: + check = c.get_check(v, var_type) + m = c.member + with writer.if_block(check, not first, False) as if_scope: + item.type = c.member.member_type + item.subprefix = item.prefix + "_" + m.name + item.non_null = c.member.has_attr("nonnull") + sub_want_extra_size = want_extra_size + if sub_want_extra_size and not m.contains_extra_size(): + writer.assign(item.extra_size(), 0) + sub_want_extra_size = False + + write_validate_item(writer, container, item, if_scope, scope, start, + want_nw_size, want_mem_size, sub_want_extra_size) + + first = False + + with writer.block(" else"): + if want_nw_size: + writer.assign(item.nw_size(), 0) + if want_mem_size: + writer.assign(item.mem_size(), 0) + if want_extra_size: + writer.assign(item.extra_size(), 0) + + writer.newline() + +def write_validate_struct_function(writer, struct): + validate_function = "validate_%s" % struct.c_type() + if writer.is_generated("validator", validate_function): + return validate_function + + writer.set_is_generated("validator", validate_function) + writer = writer.function_helper() + scope = writer.function(validate_function, "intptr_t", "uint8_t *message_start, uint8_t *message_end, SPICE_ADDRESS offset, int minor") + scope.variable_def("uint8_t *", "start = message_start + offset") + scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); + scope.variable_def("size_t", "mem_size", "nw_size"); + num_pointers = struct.get_num_pointers() + if num_pointers != 0: + scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size"); + + writer.newline() + with writer.if_block("offset == 0"): + writer.statement("return 0") + + writer.newline() + writer.error_check("start >= message_end") + + writer.newline() + write_validate_container(writer, None, struct, "start", scope, True, True, False) + + writer.newline() + writer.comment("Check if struct fits in reported side").newline() + writer.error_check("start + nw_size > message_end") + + writer.statement("return mem_size") + + writer.newline() + writer.label("error") + writer.statement("return -1") + + writer.end_block() + + return validate_function + +def write_validate_pointer_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + if want_nw_size: + writer.assign(item.nw_size(), 8) + + if want_mem_size or want_extra_size: + target_type = item.type.target_type + + v = write_read_primitive_item(writer, item, scope) + if item.non_null: + writer.error_check("%s == 0" % v) + + # pointer target is struct, or array of primitives + # if array, need no function check + + if target_type.is_array(): + writer.error_check("message_start + %s >= message_end" % v) + + + assert target_type.element_type.is_primitive() + + array_item = ItemInfo(target_type, "%s__array" % item.prefix, start) + scope.variable_def("uint32_t", array_item.nw_size()) + scope.variable_def("uint32_t", array_item.mem_size()) + if target_type.is_cstring_length(): + writer.assign(array_item.nw_size(), "spice_strnlen((char *)message_start + %s, message_end - (message_start + %s))" % (v, v)) + writer.error_check("*(message_start + %s + %s) != 0" % (v, array_item.nw_size())) + writer.assign(array_item.mem_size(), array_item.nw_size()) + else: + write_validate_array_item(writer, container, array_item, scope, parent_scope, start, + True, True, False) + writer.error_check("message_start + %s + %s > message_end" % (v, array_item.nw_size())) + + if want_extra_size: + if item.member and item.member.has_attr("nocopy"): + writer.comment("@nocopy, so no extra size").newline() + writer.assign(item.extra_size(), 0) + elif target_type.element_type.get_fixed_nw_size == 1: + writer.assign(item.extra_size(), array_item.mem_size()) + # If not bytes or zero, add padding needed for alignment + else: + writer.assign(item.extra_size(), "%s + /* for alignment */ 3" % array_item.mem_size()) + if want_mem_size: + writer.assign(item.mem_size(), "sizeof(void *) + %s" % array_item.mem_size()) + + elif target_type.is_struct(): + validate_function = write_validate_struct_function(writer, target_type) + writer.assign("ptr_size", "%s(message_start, message_end, %s, minor)" % (validate_function, v)) + writer.error_check("ptr_size < 0") + + if want_extra_size: + writer.assign(item.extra_size(), "ptr_size + /* for alignment */ 3") + if want_mem_size: + writer.assign(item.mem_size(), "sizeof(void *) + ptr_size") + else: + raise NotImplementedError("pointer to unsupported type %s" % target_type) + + +def write_validate_array_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + array = item.type + is_byte_size = False + element_type = array.element_type + if array.is_bytes_length(): + nelements = "%s__nbytes" %(item.prefix) + else: + nelements = "%s__nelements" %(item.prefix) + if not parent_scope.variable_defined(nelements): + parent_scope.variable_def("uint32_t", nelements) + + if array.is_constant_length(): + writer.assign(nelements, array.size) + elif array.is_remaining_length(): + if element_type.is_fixed_nw_size(): + if element_type.get_fixed_nw_size() == 1: + writer.assign(nelements, "message_end - %s" % item.get_position()) + else: + writer.assign(nelements, "(message_end - %s) / (%s)" %(item.get_position(), element_type.get_fixed_nw_size())) + else: + raise NotImplementedError("TODO array[] of dynamic element size not done yet") + elif array.is_identifier_length(): + v = write_read_primitive(writer, start, container, array.size, scope) + writer.assign(nelements, v) + elif array.is_image_size_length(): + bpp = array.size[1] + width = array.size[2] + rows = array.size[3] + width_v = write_read_primitive(writer, start, container, width, scope) + rows_v = write_read_primitive(writer, start, container, rows, scope) + # TODO: Handle multiplication overflow + if bpp == 8: + writer.assign(nelements, "%s * %s" % (width_v, rows_v)) + elif bpp == 1: + writer.assign(nelements, "((%s + 7) / 8 ) * %s" % (width_v, rows_v)) + else: + writer.assign(nelements, "((%s * %s + 7) / 8 ) * %s" % (bpp, width_v, rows_v)) + elif array.is_bytes_length(): + is_byte_size = True + v = write_read_primitive(writer, start, container, array.size[1], scope) + writer.assign(nelements, v) + elif array.is_cstring_length(): + writer.todo("cstring array size type not handled yet") + else: + writer.todo("array size type not handled yet") + + writer.newline() + + nw_size = item.nw_size() + mem_size = item.mem_size() + extra_size = item.extra_size() + + if is_byte_size and want_nw_size: + writer.assign(nw_size, nelements) + want_nw_size = False + + if element_type.is_fixed_nw_size() and want_nw_size: + element_size = element_type.get_fixed_nw_size() + # TODO: Overflow check the multiplication + if element_size == 1: + writer.assign(nw_size, nelements) + else: + writer.assign(nw_size, "(%s) * %s" % (element_size, nelements)) + want_nw_size = False + + if element_type.is_fixed_sizeof() and want_mem_size and not is_byte_size: + # TODO: Overflow check the multiplication + writer.assign(mem_size, "%s * %s" % (element_type.sizeof(), nelements)) + want_mem_size = False + + if not element_type.contains_extra_size() and want_extra_size: + writer.assign(extra_size, 0) + want_extra_size = False + + if not (want_mem_size or want_nw_size or want_extra_size): + return + + start2 = codegen.increment_identifier(start) + scope.variable_def("uint8_t *", "%s = %s" % (start2, item.get_position())) + if is_byte_size: + start2_end = "%s_array_end" % start2 + scope.variable_def("uint8_t *", start2_end) + + element_item = ItemInfo(element_type, "%s__element" % item.prefix, start2) + + element_nw_size = element_item.nw_size() + element_mem_size = element_item.mem_size() + scope.variable_def("uint32_t", element_nw_size) + scope.variable_def("uint32_t", element_mem_size) + + if want_nw_size: + writer.assign(nw_size, 0) + if want_mem_size: + writer.assign(mem_size, 0) + if want_extra_size: + writer.assign(extra_size, 0) + + want_element_nw_size = want_nw_size + if element_type.is_fixed_nw_size(): + start_increment = element_type.get_fixed_nw_size() + else: + want_element_nw_size = True + start_increment = element_nw_size + + if is_byte_size: + writer.assign(start2_end, "%s + %s" % (start2, nelements)) + + with writer.index(no_block = is_byte_size) as index: + with writer.while_loop("%s < %s" % (start2, start2_end) ) if is_byte_size else writer.for_loop(index, nelements) as scope: + write_validate_item(writer, container, element_item, scope, parent_scope, start2, + want_element_nw_size, want_mem_size, want_extra_size) + + if want_nw_size: + writer.increment(nw_size, element_nw_size) + if want_mem_size: + writer.increment(mem_size, element_mem_size) + if want_extra_size: + writer.increment(extra_size, element_extra_size) + + writer.increment(start2, start_increment) + if is_byte_size: + writer.error_check("%s != %s" % (start2, start2_end)) + +def write_validate_struct_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + struct = item.type + start2 = codegen.increment_identifier(start) + scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", start2 + " = %s" % (item.get_position())) + + write_validate_container(writer, item.prefix, struct, start2, scope, want_nw_size, want_mem_size, want_extra_size) + +def write_validate_primitive_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + if want_nw_size: + nw_size = item.nw_size() + writer.assign(nw_size, item.type.get_fixed_nw_size()) + if want_mem_size: + mem_size = item.mem_size() + writer.assign(mem_size, item.type.sizeof()) + assert not want_extra_size + +def write_validate_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + if item.type.is_pointer(): + write_validate_pointer_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size) + elif item.type.is_array(): + write_validate_array_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size) + elif item.type.is_struct(): + write_validate_struct_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size) + elif item.type.is_primitive(): + write_validate_primitive_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size) + else: + writer.todo("Implement validation of %s" % item.type) + +def write_validate_member(writer, container, member, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size): + if member.has_minor_attr(): + prefix = "if (minor >= %s)" % (member.get_minor_attr()) + newline = False + else: + prefix = "" + newline = True + item = MemberItemInfo(member, container, start) + with writer.block(prefix, newline=newline, comment=member.name) as scope: + if member.is_switch(): + write_validate_switch_member(writer, container, member, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size) + else: + write_validate_item(writer, container, item, scope, parent_scope, start, + want_nw_size, want_mem_size, want_extra_size) + + if member.has_minor_attr(): + with writer.block(" else", comment = "minor < %s" % (member.get_minor_attr())): + if member.is_array(): + nelements = "%s__nelements" %(item.prefix) + writer.assign(nelements, 0) + if want_nw_size: + writer.assign(item.nw_size(), 0) + + if want_mem_size: + if member.is_fixed_sizeof(): + writer.assign(item.mem_size(), member.sizeof()) + elif member.is_array(): + writer.assign(item.mem_size(), 0) + else: + raise NotImplementedError("TODO minor check for non-constant items") + + assert not want_extra_size + +def write_validate_container(writer, prefix, container, start, parent_scope, want_nw_size, want_mem_size, want_extra_size): + for m in container.members: + sub_want_nw_size = want_nw_size and not m.is_fixed_nw_size() + sub_want_mem_size = m.is_extra_size() + sub_want_extra_size = not m.is_extra_size() and m.contains_extra_size() + + defs = ["size_t"] + if sub_want_nw_size: + defs.append (m.name + "__nw_size") + if sub_want_mem_size: + defs.append (m.name + "__mem_size") + if sub_want_extra_size: + defs.append (m.name + "__extra_size") + + if sub_want_nw_size or sub_want_mem_size or sub_want_extra_size: + parent_scope.variable_def(*defs) + write_validate_member(writer, container, m, parent_scope, start, + sub_want_nw_size, sub_want_mem_size, sub_want_extra_size) + writer.newline() + + if want_nw_size: + if prefix: + nw_size = prefix + "__nw_size" + else: + nw_size = "nw_size" + + size = 0 + for m in container.members: + if m.is_fixed_nw_size(): + size = size + m.get_fixed_nw_size() + + nm_sum = str(size) + for m in container.members: + if not m.is_fixed_nw_size(): + nm_sum = nm_sum + " + " + m.name + "__nw_size" + + writer.assign(nw_size, nm_sum) + + if want_mem_size: + if prefix: + mem_size = prefix + "__mem_size" + else: + mem_size = "mem_size" + + mem_sum = container.sizeof() + for m in container.members: + if m.is_extra_size(): + mem_sum = mem_sum + " + " + m.name + "__mem_size" + elif m.contains_extra_size(): + mem_sum = mem_sum + " + " + m.name + "__extra_size" + + writer.assign(mem_size, mem_sum) + + if want_extra_size: + if prefix: + extra_size = prefix + "__extra_size" + else: + extra_size = "extra_size" + + extra_sum = [] + for m in container.members: + if m.is_extra_size(): + extra_sum.append(m.name + "__mem_size") + elif m.contains_extra_size(): + extra_sum.append(m.name + "__extra_size") + writer.assign(extra_size, codegen.sum_array(extra_sum)) + +class DemarshallingDestination: + def __init__(self): + pass + + def child_at_end(self, writer, t): + return RootDemarshallingDestination(self, t.c_type(), t.sizeof()) + + def child_sub(self, member): + return SubDemarshallingDestination(self, member) + + def declare(self, writer): + return writer.optional_block(self.reuse_scope) + + def is_toplevel(self): + return self.parent_dest == None and not self.is_helper + +class RootDemarshallingDestination(DemarshallingDestination): + def __init__(self, parent_dest, c_type, sizeof, pointer = None): + self.is_helper = False + self.reuse_scope = None + self.parent_dest = parent_dest + if parent_dest: + self.base_var = codegen.increment_identifier(parent_dest.base_var) + else: + self.base_var = "out" + self.c_type = c_type + self.sizeof = sizeof + self.pointer = pointer # None == at "end" + + def get_ref(self, member): + return self.base_var + "->" + member + + def declare(self, writer): + if self.reuse_scope: + scope = self.reuse_scope + else: + writer.begin_block() + scope = writer.get_subwriter() + + scope.variable_def(self.c_type + " *", self.base_var) + if not self.reuse_scope: + scope.newline() + + if self.pointer: + writer.assign(self.base_var, "(%s *)%s" % (self.c_type, self.pointer)) + else: + writer.assign(self.base_var, "(%s *)end" % (self.c_type)) + writer.increment("end", self.sizeof) + writer.newline() + + if self.reuse_scope: + return writer.no_block(self.reuse_scope) + else: + return writer.partial_block(scope) + +class SubDemarshallingDestination(DemarshallingDestination): + def __init__(self, parent_dest, member): + self.reuse_scope = None + self.parent_dest = parent_dest + self.base_var = parent_dest.base_var + self.member = member + self.is_helper = False + + def get_ref(self, member): + return self.parent_dest.get_ref(self.member) + "." + member + +def read_array_len(writer, prefix, array, dest, scope, handles_bytes = False): + if array.is_bytes_length(): + nelements = "%s__nbytes" % prefix + else: + nelements = "%s__nelements" % prefix + if dest.is_toplevel(): + return nelements # Already there for toplevel, need not recalculate + element_type = array.element_type + scope.variable_def("uint32_t", nelements) + if array.is_constant_length(): + writer.assign(nelements, array.size) + elif array.is_identifier_length(): + writer.assign(nelements, dest.get_ref(array.size)) + elif array.is_remaining_length(): + if element_type.is_fixed_nw_size(): + writer.assign(nelements, "(message_end - in) / (%s)" %(element_type.get_fixed_nw_size())) + else: + raise NotImplementedError("TODO array[] of dynamic element size not done yet") + elif array.is_image_size_length(): + bpp = array.size[1] + width = array.size[2] + rows = array.size[3] + width_v = dest.get_ref(width) + rows_v = dest.get_ref(rows) + # TODO: Handle multiplication overflow + if bpp == 8: + writer.assign(nelements, "%s * %s" % (width_v, rows_v)) + elif bpp == 1: + writer.assign(nelements, "((%s + 7) / 8 ) * %s" % (width_v, rows_v)) + else: + writer.assign(nelements, "((%s * %s + 7) / 8 ) * %s" % (bpp, width_v, rows_v)) + elif array.is_bytes_length(): + if not handles_bytes: + raise NotImplementedError("handling of bytes() not supported here yet") + writer.assign(nelements, dest.get_ref(array.size[1])) + else: + raise NotImplementedError("TODO array size type not handled yet") + return nelements + +def write_switch_parser(writer, container, switch, dest, scope): + var = container.lookup_member(switch.variable) + var_type = var.member_type + + if switch.has_attr("fixedsize"): + scope.variable_def("uint8_t *", "in_save") + writer.assign("in_save", "in") + + first = True + for c in switch.cases: + check = c.get_check(dest.get_ref(switch.variable), var_type) + m = c.member + with writer.if_block(check, not first, False) as block: + t = m.member_type + if switch.has_end_attr(): + dest2 = dest.child_at_end(writer, m.member_type) + elif switch.has_attr("anon"): + dest2 = dest + else: + if t.is_struct(): + dest2 = dest.child_sub(switch.name + "." + m.name) + else: + dest2 = dest.child_sub(switch.name) + dest2.reuse_scope = block + + if t.is_struct(): + write_container_parser(writer, t, dest2) + elif t.is_pointer(): + write_parse_pointer(writer, t, False, dest2, m.name, not m.has_attr("ptr32"), block) + elif t.is_primitive(): + writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) + #TODO validate e.g. flags and enums + elif t.is_array(): + nelements = read_array_len(writer, m.name, t, dest, block) + write_array_parser(writer, nelements, t, dest, block) + else: + writer.todo("Can't handle type %s" % m.member_type) + + first = False + + writer.newline() + + if switch.has_attr("fixedsize"): + writer.assign("in", "in_save + %s" % switch.get_fixed_nw_size()) + +def write_parse_ptr_function(writer, target_type): + if target_type.is_array(): + parse_function = "parse_array_%s" % target_type.element_type.primitive_type() + else: + parse_function = "parse_struct_%s" % target_type.c_type() + if writer.is_generated("parser", parse_function): + return parse_function + + writer.set_is_generated("parser", parse_function) + + writer = writer.function_helper() + scope = writer.function(parse_function, "uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *this_ptr_info, int minor") + scope.variable_def("uint8_t *", "in = message_start + this_ptr_info->offset") + scope.variable_def("uint8_t *", "end") + + num_pointers = target_type.get_num_pointers() + if num_pointers != 0: + scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size"); + scope.variable_def("uint32_t", "n_ptr=0"); + scope.variable_def("PointerInfo", "ptr_info[%s]" % num_pointers) + + writer.newline() + if target_type.is_array(): + writer.assign("end", "struct_data") + else: + writer.assign("end", "struct_data + %s" % (target_type.sizeof())) + + dest = RootDemarshallingDestination(None, target_type.c_type(), target_type.sizeof(), "struct_data") + dest.is_helper = True + dest.reuse_scope = scope + if target_type.is_array(): + write_array_parser(writer, "this_ptr_info->nelements", target_type, dest, scope) + else: + write_container_parser(writer, target_type, dest) + + if num_pointers != 0: + write_ptr_info_check(writer) + + writer.statement("return end") + + if writer.has_error_check: + writer.newline() + writer.label("error") + writer.statement("return NULL") + + writer.end_block() + + return parse_function + +def write_array_parser(writer, nelements, array, dest, scope): + is_byte_size = array.is_bytes_length() + + element_type = array.element_type + if element_type == ptypes.uint8 or element_type == ptypes.int8: + writer.statement("memcpy(end, in, %s)" % (nelements)) + writer.increment("in", nelements) + writer.increment("end", nelements) + else: + if is_byte_size: + scope.variable_def("uint8_t *", "array_end") + writer.assign("array_end", "end + %s" % nelements) + with writer.index(no_block = is_byte_size) as index: + with writer.while_loop("end < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: + if element_type.is_primitive(): + writer.statement("*(%s *)end = consume_%s(&in)" % (element_type.c_type(), element_type.primitive_type())) + writer.increment("end", element_type.sizeof()) + else: + dest2 = dest.child_at_end(writer, element_type) + dest2.reuse_scope = array_scope + write_container_parser(writer, element_type, dest2) + +def write_parse_pointer(writer, t, at_end, dest, member_name, is_64bit, scope): + target_type = t.target_type + if is_64bit: + writer.assign("ptr_info[n_ptr].offset", "consume_uint64(&in)") + else: + writer.assign("ptr_info[n_ptr].offset", "consume_uint32(&in)") + writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) + if at_end: + writer.assign("ptr_info[n_ptr].dest", "end") + writer.increment("end", "sizeof(SPICE_ADDRESS)"); + else: + writer.assign("ptr_info[n_ptr].dest", "&%s" % dest.get_ref(member_name)) + if target_type.is_array(): + nelements = read_array_len(writer, member_name, target_type, dest, scope) + writer.assign("ptr_info[n_ptr].nelements", nelements) + + writer.statement("n_ptr++") + +def write_member_parser(writer, container, member, dest, scope): + if member.is_switch(): + write_switch_parser(writer, container, member, dest, scope) + return + + t = member.member_type + + if t.is_pointer(): + if member.has_attr("nocopy"): + writer.comment("Reuse data from network message").newline() + writer.assign(dest.get_ref(member.name), "(size_t)(message_start + consume_uint64(&in))") + else: + write_parse_pointer(writer, t, member.has_end_attr(), dest, member.name, not member.has_attr("ptr32"), scope) + elif t.is_primitive(): + if member.has_end_attr(): + writer.statement("*(%s *)end = consume_%s(&in)" % (t.c_type(), t.primitive_type())) + writer.increment("end", t.sizeof()) + else: + writer.assign(dest.get_ref(member.name), "consume_%s(&in)" % (t.primitive_type())) + #TODO validate e.g. flags and enums + elif t.is_array(): + nelements = read_array_len(writer, member.name, t, dest, scope, handles_bytes = True) + write_array_parser(writer, nelements, t, dest, scope) + elif t.is_struct(): + if member.has_end_attr(): + dest2 = dest.child_at_end(writer, t) + else: + dest2 = dest.child_sub(member.name) + writer.comment(member.name) + write_container_parser(writer, t, dest2) + else: + raise NotImplementedError("TODO can't handle parsing of %s" % t) + +def write_container_parser(writer, container, dest): + with dest.declare(writer) as scope: + for m in container.members: + if m.has_minor_attr(): + writer.begin_block("if (minor >= %s)" % m.get_minor_attr()) + write_member_parser(writer, container, m, dest, scope) + if m.has_minor_attr(): + # We need to zero out the fixed part of all optional fields + if not m.member_type.is_array(): + writer.end_block(newline=False) + writer.begin_block(" else") + # TODO: This is not right for fields that don't exist in the struct + if m.member_type.is_primitive(): + writer.assign(dest.get_ref(m.name), "0") + elif m.is_fixed_sizeof(): + writer.statement("memset ((char *)&%s, 0, %s)" % (dest.get_ref(m.name), m.sizeof())) + else: + raise NotImplementedError("TODO Clear optional dynamic fields") + writer.end_block() + +def write_ptr_info_check(writer): + writer.newline() + with writer.index() as index: + with writer.for_loop(index, "n_ptr") as scope: + offset = "ptr_info[%s].offset" % index + function = "ptr_info[%s].parse" % index + dest = "ptr_info[%s].dest" % index + with writer.if_block("%s == 0" % offset, newline=False): + writer.assign("*%s" % dest, "0") + with writer.block(" else"): + writer.comment("Align to 32 bit").newline() + writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") + writer.assign("*%s" % dest, "(size_t)end") + writer.assign("end", "%s(message_start, message_end, end, &ptr_info[%s], minor)" % (function, index)) + writer.error_check("end == NULL") + writer.newline() + +def write_msg_parser(writer, message): + msg_name = message.c_name() + function_name = "parse_%s" % msg_name + if writer.is_generated("demarshaller", function_name): + return function_name + writer.set_is_generated("demarshaller", function_name) + + msg_type = message.c_type() + msg_sizeof = message.sizeof() + + writer.newline() + parent_scope = writer.function(function_name, + "uint8_t *", + "uint8_t *message_start, uint8_t *message_end, int minor, size_t *size", True) + parent_scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); + parent_scope.variable_def("uint8_t *", "start = message_start"); + parent_scope.variable_def("uint8_t *", "data = NULL"); + parent_scope.variable_def("size_t", "mem_size", "nw_size"); + if not message.has_attr("nocopy"): + parent_scope.variable_def("uint8_t *", "in", "end"); + num_pointers = message.get_num_pointers() + if num_pointers != 0: + parent_scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size"); + parent_scope.variable_def("uint32_t", "n_ptr=0"); + parent_scope.variable_def("PointerInfo", "ptr_info[%s]" % num_pointers) + writer.newline() + + write_parser_helpers(writer) + + write_validate_container(writer, None, message, "start", parent_scope, True, True, False) + + writer.newline() + + writer.comment("Check if message fits in reported side").newline() + with writer.block("if (start + nw_size > message_end)"): + writer.statement("return NULL") + + writer.newline().comment("Validated extents and calculated size").newline() + + if message.has_attr("nocopy"): + writer.assign("data", "message_start") + writer.assign("*size", "message_end - message_start") + else: + writer.assign("data", "(uint8_t *)malloc(mem_size)") + writer.error_check("data == NULL") + writer.assign("end", "data + %s" % (msg_sizeof)) + writer.assign("in", "start").newline() + + dest = RootDemarshallingDestination(None, msg_type, msg_sizeof, "data") + dest.reuse_scope = parent_scope + write_container_parser(writer, message, dest) + + writer.newline() + writer.statement("assert(in <= message_end)") + + if num_pointers != 0: + write_ptr_info_check(writer) + + writer.statement("assert(end <= data + mem_size)") + + writer.newline() + writer.assign("*size", "end - data") + + writer.statement("return data") + writer.newline() + if writer.has_error_check: + writer.label("error") + with writer.block("if (data != NULL)"): + writer.statement("free(data)") + writer.statement("return NULL") + writer.end_block() + + return function_name + +def write_channel_parser(writer, channel, server): + writer.newline() + ids = {} + min_id = 1000000 + if server: + messages = channel.server_messages + else: + messages = channel.client_messages + for m in messages: + ids[m.value] = m + + ranges = [] + ids2 = ids.copy() + while len(ids2) > 0: + end = start = min(ids2.keys()) + while ids2.has_key(end): + del ids2[end] + end = end + 1 + + ranges.append( (start, end) ) + + if server: + function_name = "parse_%s_msg" % channel.name + else: + function_name = "parse_%s_msgc" % channel.name + writer.newline() + scope = writer.function(function_name, + "uint8_t *", + "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out") + + helpers = writer.function_helper() + + d = 0 + for r in ranges: + d = d + 1 + writer.write("static parse_msg_func_t funcs%d[%d] = " % (d, r[1] - r[0])); + writer.begin_block() + for i in range(r[0], r[1]): + func = write_msg_parser(helpers, ids[i].message_type) + writer.write(func) + if i != r[1] -1: + writer.write(",") + writer.newline() + + writer.end_block(semicolon = True) + + d = 0 + for r in ranges: + d = d + 1 + with writer.if_block("message_type >= %d && message_type < %d" % (r[0], r[1]), d > 1, False): + writer.statement("return funcs%d[message_type-%d](message_start, message_end, minor, size_out)" % (d, r[0])) + writer.newline() + + writer.statement("return NULL") + writer.end_block() + + return function_name + +def write_get_channel_parser(writer, channel_parsers, max_channel, is_server): + writer.newline() + if is_server: + function_name = "spice_get_server_channel_parser" + else: + function_name = "spice_get_client_channel_parser" + + scope = writer.function(function_name, + "spice_parse_channel_func_t", + "uint32_t channel, unsigned int *max_message_type") + + writer.write("static struct {spice_parse_channel_func_t func; unsigned int max_messages; } channels[%d] = " % (max_channel+1)) + writer.begin_block() + for i in range(0, max_channel + 1): + writer.write("{ ") + if channel_parsers.has_key(i): + writer.write(channel_parsers[i][1]) + writer.write(", ") + + channel = channel_parsers[i][0] + max_msg = 0 + if is_server: + messages = channel.server_messages + else: + messages = channel.client_messages + for m in messages: + max_msg = max(max_msg, m.value) + writer.write(max_msg) + else: + writer.write("NULL, 0") + writer.write("}") + + if i != max_channel: + writer.write(",") + writer.newline() + writer.end_block(semicolon = True) + + with writer.if_block("channel < %d" % (max_channel + 1)): + with writer.if_block("max_message_type != NULL"): + writer.assign("*max_message_type", "channels[channel].max_messages") + writer.statement("return channels[channel].func") + + writer.statement("return NULL") + writer.end_block() + + +def write_full_protocol_parser(writer, is_server): + writer.newline() + if is_server: + function_name = "spice_parse_msg" + else: + function_name = "spice_parse_reply" + scope = writer.function(function_name, + "uint8_t *", + "uint8_t *message_start, uint8_t *message_end, uint32_t channel, uint16_t message_type, int minor, size_t *size_out") + scope.variable_def("spice_parse_channel_func_t", "func" ) + + if is_server: + writer.assign("func", "spice_get_server_channel_parser(channel, NULL)") + else: + writer.assign("func", "spice_get_client_channel_parser(channel, NULL)") + + with writer.if_block("func != NULL"): + writer.statement("return func(message_start, message_end, message_type, minor, size_out)") + + writer.statement("return NULL") + writer.end_block() + +def write_protocol_parser(writer, proto, is_server): + max_channel = 0 + parsers = {} + + for channel in proto.channels: + max_channel = max(max_channel, channel.value) + + parsers[channel.value] = (channel.channel_type, write_channel_parser(writer, channel.channel_type, is_server)) + + write_get_channel_parser(writer, parsers, max_channel, is_server) + write_full_protocol_parser(writer, is_server) + +def write_includes(writer): + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.newline() + writer.writeln("#ifdef _MSC_VER") + writer.writeln("#pragma warning(disable:4101)") + writer.writeln("#endif") diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py new file mode 100644 index 0000000..fe8a321 --- /dev/null +++ b/python_modules/ptypes.py @@ -0,0 +1,965 @@ +import codegen +import types + +_types_by_name = {} +_types = [] + +def type_exists(name): + return _types_by_name.has_key(name) + +def lookup_type(name): + return _types_by_name[name] + +def get_named_types(): + return _types + +class FixedSize: + def __init__(self, val = 0, minor = 0): + if isinstance(val, FixedSize): + self.vals = val.vals + else: + self.vals = [0] * (minor + 1) + self.vals[minor] = val + + def __add__(self, other): + if isinstance(other, types.IntType): + other = FixedSize(other) + + new = FixedSize() + l = max(len(self.vals), len(other.vals)) + shared = min(len(self.vals), len(other.vals)) + + new.vals = [0] * l + + for i in range(shared): + new.vals[i] = self.vals[i] + other.vals[i] + + for i in range(shared,len(self.vals)): + new.vals[i] = self.vals[i]; + + for i in range(shared,len(other.vals)): + new.vals[i] = new.vals[i] + other.vals[i]; + + return new + + def __radd__(self, other): + return self.__add__(other) + + def __str__(self): + s = "%d" % (self.vals[0]) + + for i in range(1,len(self.vals)): + if self.vals[i] > 0: + s = s + " + ((minor >= %d)?%d:0)" % (i, self.vals[i]) + return s + +class Type: + def __init__(self): + self.attributes = {} + self.registred = False + self.name = None + + def has_name(self): + return self.name != None + + def get_type(self, recursive=False): + return self + + def is_primitive(self): + return False + + def is_fixed_sizeof(self): + return True + + def is_extra_size(self): + return False + + def contains_extra_size(self): + return False + + def is_fixed_nw_size(self): + return True + + def is_array(self): + return isinstance(self, ArrayType) + + def is_struct(self): + return isinstance(self, StructType) + + def is_pointer(self): + return isinstance(self, PointerType) + + def get_num_pointers(self): + return 0 + + def get_pointer_names(self): + return [] + + def sizeof(self): + return "sizeof(%s)" % (self.c_type()) + + def __repr__(self): + return self.__str__() + + def __str__(self): + if self.name != None: + return self.name + return "anonymous type" + + def resolve(self): + return self + + def register(self): + if self.registred or self.name == None: + return + self.registred = True + if _types_by_name.has_key(self.name): + raise Exception, "Type %s already defined" % self.name + _types.append(self) + _types_by_name[self.name] = self + + def has_pointer(self): + return False + + def has_attr(self, name): + return self.attributes.has_key(name) + +class TypeRef(Type): + def __init__(self, name): + Type.__init__(self) + self.name = name + + def __str__(self): + return "ref to %s" % (self.name) + + def resolve(self): + if not _types_by_name.has_key(self.name): + raise Exception, "Unknown type %s" % self.name + return _types_by_name[self.name] + + def register(self): + assert True, "Can't register TypeRef!" + + +class IntegerType(Type): + def __init__(self, bits, signed): + Type.__init__(self) + self.bits = bits + self.signed = signed + + if signed: + self.name = "int%d" % bits + else: + self.name = "uint%d" % bits + + def primitive_type(self): + return self.name + + def c_type(self): + return self.name + "_t" + + def get_fixed_nw_size(self): + return self.bits / 8 + + def is_primitive(self): + return True + +class TypeAlias(Type): + def __init__(self, name, the_type, attribute_list): + Type.__init__(self) + self.name = name + self.the_type = the_type + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def get_type(self, recursive=False): + if recursive: + return self.the_type.get_type(True) + else: + return self.the_type + + def primitive_type(self): + return self.the_type.primitive_type() + + def resolve(self): + self.the_type = self.the_type.resolve() + return self + + def __str__(self): + return "alias %s" % self.name + + def is_primitive(self): + return self.the_type.is_primitive() + + def is_fixed_sizeof(self): + return self.the_type.is_fixed_sizeof() + + def is_fixed_nw_size(self): + return self.the_type.is_fixed_nw_size() + + def get_fixed_nw_size(self): + return self.the_type.get_fixed_nw_size() + + def get_num_pointers(self): + return self.the_type.get_num_pointers() + + def get_pointer_names(self): + return self.the_type.get_pointer_names() + + def c_type(self): + if self.has_attr("ctype"): + return self.attributes["ctype"][0] + return self.name + + def has_pointer(self): + return self.the_type.has_pointer() + +class EnumBaseType(Type): + def is_enum(self): + return isinstance(self, EnumType) + + def primitive_type(self): + return "uint%d" % (self.bits) + + def c_type(self): + return "uint%d_t" % (self.bits) + + def c_name(self): + return codegen.prefix_camel(self.name) + + def c_enumname(self, value): + if self.has_attr("prefix"): + return self.attributes["prefix"][0] + self.names[value] + return codegen.prefix_underscore_upper(self.name.upper(), self.names[value]) + + def c_enumname_by_name(self, name): + if self.has_attr("prefix"): + return self.attributes["prefix"][0] + self.names[value] + return codegen.prefix_underscore_upper(self.name.upper(), name) + + def is_primitive(self): + return True + + def get_fixed_nw_size(self): + return self.bits / 8 + +class EnumType(EnumBaseType): + def __init__(self, bits, name, enums, attribute_list): + Type.__init__(self) + self.bits = bits + self.name = name + + last = -1 + names = {} + values = {} + for v in enums: + name = v[0] + if len(v) > 1: + value = v[1] + else: + value = last + 1 + last = value + + assert not names.has_key(value) + names[value] = name + values[name] = value + + self.names = names + self.values = values + + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def __str__(self): + return "enum %s" % self.name + + def c_define(self, writer): + writer.write("enum ") + writer.write(self.c_name()) + writer.begin_block() + values = self.names.keys() + values.sort() + current_default = 0 + for i in values: + writer.write(self.c_enumname(i)) + if i != current_default: + writer.write(" = %d" % (i)) + writer.write(",") + writer.newline() + current_default = i + 1 + writer.newline() + writer.write(codegen.prefix_underscore_upper(self.name.upper(), "ENUM_END")) + writer.newline() + writer.end_block(semicolon=True) + writer.newline() + +class FlagsType(EnumBaseType): + def __init__(self, bits, name, flags, attribute_list): + Type.__init__(self) + self.bits = bits + self.name = name + + last = -1 + names = {} + values = {} + for v in flags: + name = v[0] + if len(v) > 1: + value = v[1] + else: + value = last + 1 + last = value + + assert not names.has_key(value) + names[value] = name + values[name] = value + + self.names = names + self.values = values + + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def __str__(self): + return "flags %s" % self.name + + def c_define(self, writer): + writer.write("enum ") + writer.write(self.c_name()) + writer.begin_block() + values = self.names.keys() + values.sort() + mask = 0 + for i in values: + writer.write(self.c_enumname(i)) + mask = mask | (1< 0 + + def is_image_size_length(self): + if isinstance(self.size, types.IntType) or isinstance(self.size, types.StringType): + return False + return self.size[0] == "image_size" + + def is_bytes_length(self): + if isinstance(self.size, types.IntType) or isinstance(self.size, types.StringType): + return False + return self.size[0] == "bytes" + + def is_cstring_length(self): + if isinstance(self.size, types.IntType) or isinstance(self.size, types.StringType): + return False + return self.size[0] == "cstring" + + def is_fixed_sizeof(self): + return self.is_constant_length() and self.element_type.is_fixed_sizeof() + + def is_fixed_nw_size(self): + return self.is_constant_length() and self.element_type.is_fixed_nw_size() + + def get_fixed_nw_size(self): + if not self.is_fixed_nw_size(): + raise Exception, "Not a fixed size type" + + return self.element_type.get_fixed_nw_size() * self.size + + def get_num_pointers(self): + element_count = self.element_type.get_num_pointers() + if element_count == 0: + return 0 + if self.is_constant_length(self): + return element_count * self.size + raise Exception, "Pointers in dynamic arrays not supported" + + def get_pointer_names(self): + element_count = self.element_type.get_num_pointers() + if element_count == 0: + return [] + raise Exception, "Pointer names in arrays not supported" + + def contains_extra_size(self): + return self.element_type.contains_extra_size() + + def sizeof(self): + return "%s * %s" % (self.element_type.sizeof(), self.size) + + def c_type(self): + return self.element_type.c_type() + +class PointerType(Type): + def __init__(self, target_type): + Type.__init__(self) + self.name = None + self.target_type = target_type + + def __str__(self): + return "%s*" % (str(self.target_type)) + + def resolve(self): + self.target_type = self.target_type.resolve() + return self + + def get_fixed_size(self): + return 8 # offsets are 64bit + + def is_fixed_nw_size(self): + return True + + def is_primitive(self): + return True + + def primitive_type(self): + return "uint64" + + def get_fixed_nw_size(self): + return 8 + + def c_type(self): + return "SPICE_ADDRESS" + + def has_pointer(self): + return True + + def contains_extra_size(self): + return True + + def get_num_pointers(self): + return 1 + +class Containee: + def __init__(self): + self.attributes = {} + + def is_switch(self): + return False + + def is_pointer(self): + return not self.is_switch() and self.member_type.is_pointer() + + def is_array(self): + return not self.is_switch() and self.member_type.is_array() + + def is_struct(self): + return not self.is_switch() and self.member_type.is_struct() + + def is_primitive(self): + return not self.is_switch() and self.member_type.is_primitive() + + def has_attr(self, name): + return self.attributes.has_key(name) + + def has_minor_attr(self): + return self.has_attr("minor") + + def has_end_attr(self): + return self.has_attr("end") + + def get_minor_attr(self): + return self.attributes["minor"][0] + +class Member(Containee): + def __init__(self, name, member_type, attribute_list): + Containee.__init__(self) + self.name = name + self.member_type = member_type + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def resolve(self, container): + self.container = container + self.member_type = self.member_type.resolve() + self.member_type.register() + return self + + def is_primitive(self): + return self.member_type.is_primitive() + + def is_fixed_sizeof(self): + if self.has_end_attr(): + return False + return self.member_type.is_fixed_sizeof() + + def is_extra_size(self): + return self.has_end_attr() + + def is_fixed_nw_size(self): + return self.member_type.is_fixed_nw_size() + + def get_fixed_nw_size(self): + size = self.member_type.get_fixed_nw_size() + if self.has_minor_attr(): + minor = self.get_minor_attr() + size = FixedSize(size, minor) + return size + + def contains_extra_size(self): + return self.member_type.contains_extra_size() + + def sizeof(self): + return self.member_type.sizeof() + + def __repr__(self): + return "%s (%s)" % (str(self.name), str(self.member_type)) + + def has_pointer(self): + return self.member_type.has_pointer() + + def get_num_pointers(self): + return self.member_type.get_num_pointers() + + def get_pointer_names(self): + if self.member_type.is_pointer(): + names = [self.name + "_out"] + else: + names = self.member_type.get_pointer_names() + if self.has_attr("outvar"): + prefix = self.attributes["outvar"][0] + names = map(lambda name: prefix + "_" + name, names) + return names + +class SwitchCase: + def __init__(self, values, member): + self.values = values + self.member = member + self.members = [member] + + def get_check(self, var_cname, var_type): + checks = [] + for v in self.values: + if v == None: + return "1" + elif var_type.is_enum(): + checks.append("%s == %s" % (var_cname, var_type.c_enumname_by_name(v))) + else: + checks.append("(%s & %s)" % (var_cname, var_type.c_enumname_by_name(v))) + return " || ".join(checks) + + def resolve(self, container): + self.switch = container + self.member = self.member.resolve(self) + return self + + def has_pointer(self): + return self.member.has_pointer() + + def get_num_pointers(self): + return self.member.get_num_pointers() + + def get_pointer_names(self): + return self.member.get_pointer_names() + +class Switch(Containee): + def __init__(self, variable, cases, name, attribute_list): + Containee.__init__(self) + self.variable = variable + self.name = name + self.cases = cases + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def is_switch(self): + return True + + def has_switch_member(self, member): + for c in self.cases: + if c.member == member: + return True + return False + + def resolve(self, container): + self.container = container + self.cases = map(lambda c : c.resolve(self), self.cases) + return self + + def __repr__(self): + return "switch on %s %s" % (str(self.variable),str(self.name)) + + def is_fixed_sizeof(self): + # Kinda weird, but we're unlikely to have a real struct if there is an @end + if self.has_end_attr(): + return False + return True + + def is_fixed_nw_size(self): + if self.has_attr("fixedsize"): + return True + + size = None + for c in self.cases: + if not c.member.is_fixed_nw_size(): + return False + if size == None: + size = c.member.get_fixed_nw_size() + elif size != c.member.get_fixed_nw_size(): + return False + return True + + def is_extra_size(self): + return self.has_end_attr() + + def contains_extra_size(self): + for c in self.cases: + if c.member.is_extra_size(): + return True + if c.member.contains_extra_size(): + return True + return False + + def get_fixed_nw_size(self): + if not self.is_fixed_nw_size(): + raise Exception, "Not a fixed size type" + size = 0; + for c in self.cases: + size = max(size, c.member.get_fixed_nw_size()) + return size + + def sizeof(self): + return "sizeof(((%s *)NULL)->%s)" % (self.container.c_type(), + self.name) + + def has_pointer(self): + for c in self.cases: + if c.has_pointer(): + return True + return False + + def get_num_pointers(self): + count = 0 + for c in self.cases: + count = max(count, c.get_num_pointers()) + return count + + def get_pointer_names(self): + names = [] + for c in self.cases: + names = names + c.get_pointer_names() + return names + +class ContainerType(Type): + def is_fixed_sizeof(self): + for m in self.members: + if not m.is_fixed_sizeof(): + return False + return True + + def contains_extra_size(self): + for m in self.members: + if m.is_extra_size(): + return True + if m.contains_extra_size(): + return True + return False + + def is_fixed_nw_size(self): + for i in self.members: + if not i.is_fixed_nw_size(): + return False + return True + + def get_fixed_nw_size(self): + size = 0 + for i in self.members: + size = size + i.get_fixed_nw_size() + return size + + def get_fixed_nw_offset(self, member): + size = 0 + for i in self.members: + if i == member: + break + if i.is_fixed_nw_size(): + size = size + i.get_fixed_nw_size() + return size + + def resolve(self): + self.members = map(lambda m : m.resolve(self), self.members) + return self + + def get_num_pointers(self): + count = 0 + for m in self.members: + count = count + m.get_num_pointers() + return count + + def get_pointer_names(self): + names = [] + for m in self.members: + names = names + m.get_pointer_names() + return names + + def has_pointer(self): + for m in self.members: + if m.has_pointer(): + return True + return False + + def get_nw_offset(self, member, prefix = "", postfix = ""): + fixed = self.get_fixed_nw_offset(member) + v = [] + for m in self.members: + if m == member: + break + if m.is_switch() and m.has_switch_member(member): + break + if not m.is_fixed_nw_size(): + v.append(prefix + m.name + postfix) + if len(v) > 0: + return str(fixed) + " + " + (" + ".join(v)) + else: + return str(fixed) + + def lookup_member(self, name): + return self.members_by_name[name] + +class StructType(ContainerType): + def __init__(self, name, members, attribute_list): + Type.__init__(self) + self.name = name + self.members = members + self.members_by_name = {} + for m in members: + self.members_by_name[m.name] = m + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def __str__(self): + if self.name == None: + return "anonymous struct" + else: + return "struct %s" % self.name + + def c_type(self): + if self.has_attr("ctype"): + return self.attributes["ctype"][0] + return codegen.prefix_camel(self.name) + +class MessageType(ContainerType): + def __init__(self, name, members, attribute_list): + Type.__init__(self) + self.name = name + self.members = members + self.members_by_name = {} + for m in members: + self.members_by_name[m.name] = m + self.reverse_members = {} # ChannelMembers referencing this message + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] + + def __str__(self): + if self.name == None: + return "anonymous message" + else: + return "message %s" % self.name + + def c_name(self): + if self.name == None: + cms = self.reverse_members.keys() + if len(cms) != 1: + raise "Unknown typename for message" + cm = cms[0] + channelname = cm.channel.member_name + if channelname == None: + channelname = "" + else: + channelname = channelname + "_" + if cm.is_server: + return "msg_" + channelname + cm.name + else: + return "msgc_" + channelname + cm.name + else: + return codegen.prefix_camel("Msg", self.name) + + def c_type(self): + if self.has_attr("ctype"): + return self.attributes["ctype"][0] + if self.name == None: + cms = self.reverse_members.keys() + if len(cms) != 1: + raise "Unknown typename for message" + cm = cms[0] + channelname = cm.channel.member_name + if channelname == None: + channelname = "" + if cm.is_server: + return codegen.prefix_camel("Msg", channelname, cm.name) + else: + return codegen.prefix_camel("Msgc", channelname, cm.name) + else: + return codegen.prefix_camel("Msg", self.name) + +class ChannelMember(Containee): + def __init__(self, name, message_type, value): + Containee.__init__(self) + self.name = name + self.message_type = message_type + self.value = value + + def resolve(self, channel): + self.channel = channel + self.message_type = self.message_type.resolve() + self.message_type.reverse_members[self] = 1 + + return self + + def __repr__(self): + return "%s (%s)" % (str(self.name), str(self.message_type)) + +class ChannelType(Type): + def __init__(self, name, base, members): + Type.__init__(self) + self.name = name + self.base = base + self.member_name = None + self.members = members + + def __str__(self): + if self.name == None: + return "anonymous channel" + else: + return "channel %s" % self.name + + def is_fixed_nw_size(self): + return False + + def get_client_message(self, name): + return self.client_messages_byname[name] + + def get_server_message(self, name): + return self.server_messages_byname[name] + + def resolve(self): + if self.base != None: + self.base = self.base.resolve() + + server_messages = self.base.server_messages[:] + server_messages_byname = self.base.server_messages_byname.copy() + client_messages = self.base.client_messages[:] + client_messages_byname = self.base.client_messages_byname.copy() + else: + server_messages = [] + server_messages_byname = {} + client_messages = [] + client_messages_byname = {} + + server_count = 1 + client_count = 1 + + server = True + for m in self.members: + if m == "server": + server = True + elif m == "client": + server = False + elif server: + m.is_server = True + m = m.resolve(self) + if m.value: + server_count = m.value + 1 + else: + m.value = server_count + server_count = server_count + 1 + server_messages.append(m) + server_messages_byname[m.name] = m + else: + m.is_server = False + m = m.resolve(self) + if m.value: + client_count = m.value + 1 + else: + m.value = client_count + client_count = client_count + 1 + client_messages.append(m) + client_messages_byname[m.name] = m + + self.server_messages = server_messages + self.server_messages_byname = server_messages_byname + self.client_messages = client_messages + self.client_messages_byname = client_messages_byname + + return self + +class ProtocolMember: + def __init__(self, name, channel_type, value): + self.name = name + self.channel_type = channel_type + self.value = value + + def resolve(self, protocol): + self.channel_type = self.channel_type.resolve() + assert(self.channel_type.member_name == None) + self.channel_type.member_name = self.name + return self + + def __repr__(self): + return "%s (%s)" % (str(self.name), str(self.channel_type)) + +class ProtocolType(Type): + def __init__(self, name, channels): + Type.__init__(self) + self.name = name + self.channels = channels + + def __str__(self): + if self.name == None: + return "anonymous protocol" + else: + return "protocol %s" % self.name + + def is_fixed_nw_size(self): + return False + + def resolve(self): + count = 1 + for m in self.channels: + m = m.resolve(self) + if m.value: + count = m.value + 1 + else: + m.value = count + count = count + 1 + + return self + +int8 = IntegerType(8, True) +uint8 = IntegerType(8, False) +int16 = IntegerType(16, True) +uint16 = IntegerType(16, False) +int32 = IntegerType(32, True) +uint32 = IntegerType(32, False) +int64 = IntegerType(64, True) +uint64 = IntegerType(64, False) diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py new file mode 100644 index 0000000..65916b3 --- /dev/null +++ b/python_modules/spice_parser.py @@ -0,0 +1,157 @@ +from pyparsing import Literal, CaselessLiteral, Word, OneOrMore, ZeroOrMore, \ + Forward, delimitedList, Group, Optional, Combine, alphas, nums, restOfLine, cStyleComment, \ + alphanums, ParseException, ParseResults, Keyword, StringEnd, replaceWith + +import ptypes +import sys + +cvtInt = lambda toks: int(toks[0]) + +def parseVariableDef(toks): + t = toks[0][0] + pointer = toks[0][1] + name = toks[0][2] + array_size = toks[0][3] + attributes = toks[0][4] + + if array_size != None: + t = ptypes.ArrayType(t, array_size) + + if pointer != None: + t = ptypes.PointerType(t); + + return ptypes.Member(name, t, attributes) + +bnf = None +def SPICE_BNF(): + global bnf + + if not bnf: + + # punctuation + colon = Literal(":").suppress() + lbrace = Literal("{").suppress() + rbrace = Literal("}").suppress() + lbrack = Literal("[").suppress() + rbrack = Literal("]").suppress() + lparen = Literal("(").suppress() + rparen = Literal(")").suppress() + equals = Literal("=").suppress() + comma = Literal(",").suppress() + semi = Literal(";").suppress() + + # primitive types + int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8)) + uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8)) + int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16)) + uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16)) + int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32)) + uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32)) + int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64)) + uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64)) + + # keywords + channel_ = Keyword("channel") + enum32_ = Keyword("enum32").setParseAction(replaceWith(32)) + enum16_ = Keyword("enum16").setParseAction(replaceWith(16)) + enum8_ = Keyword("enum8").setParseAction(replaceWith(8)) + flags32_ = Keyword("flags32").setParseAction(replaceWith(32)) + flags16_ = Keyword("flags16").setParseAction(replaceWith(16)) + flags8_ = Keyword("flags8").setParseAction(replaceWith(8)) + channel_ = Keyword("channel") + server_ = Keyword("server") + client_ = Keyword("client") + protocol_ = Keyword("protocol") + typedef_ = Keyword("typedef") + struct_ = Keyword("struct") + message_ = Keyword("message") + image_size_ = Keyword("image_size") + bytes_ = Keyword("bytes") + cstring_ = Keyword("cstring") + switch_ = Keyword("switch") + default_ = Keyword("default") + case_ = Keyword("case") + + identifier = Word( alphas, alphanums + "_" ) + enumname = Word( alphanums + "_" ) + + integer = ( Combine( CaselessLiteral("0x") + Word( nums+"abcdefABCDEF" ) ) | + Word( nums+"+-", nums ) ).setName("int").setParseAction(cvtInt) + + typename = identifier.copy().setParseAction(lambda toks : ptypes.TypeRef(str(toks[0]))) + + # This is just normal "types", i.e. not channels or messages + typeSpec = Forward() + + attributeValue = integer ^ identifier + attribute = Group(Combine ("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen)) + attributes = Group(ZeroOrMore(attribute)) + arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen) + arraySizeSpecBytes = Group(bytes_ + lparen + identifier + rparen) + arraySizeSpecCString = Group(cstring_ + lparen + rparen) + arraySizeSpec = lbrack + Optional(identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^arraySizeSpecCString, default="") + rbrack + variableDef = Group(typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi) \ + .setParseAction(parseVariableDef) + + switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | case_.suppress() + identifier + colon)) + variableDef) \ + .setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) + switchBody = Group(switch_ + lparen + identifier + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \ + .setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) + messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace) + structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction(lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3])) + + # have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type" + typeSpec << ( structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ + int32_ ^ uint32_ ^ int64_ ^ uint64_ ^ + typename).setName("type") + + flagsBody = enumBody = Group(lbrace + delimitedList(Group (enumname + Optional(equals + integer))) + Optional(comma) + rbrace) + + messageSpec = Group(message_ + messageBody + attributes).setParseAction(lambda toks: ptypes.MessageType(None, toks[0][1], toks[0][2])) | typename + + channelParent = Optional(colon + typename, default=None) + channelMessage = Group(messageSpec + identifier + Optional(equals + integer, default=None) + semi) \ + .setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2])) + channelBody = channelParent + Group(lbrace + ZeroOrMore( server_ + colon | client_ + colon | channelMessage) + rbrace) + + enum_ = (enum32_ | enum16_ | enum8_) + flags_ = (flags32_ | flags16_ | flags8_) + enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction(lambda toks: ptypes.EnumType(toks[0][0], toks[0][1], toks[0][2], toks[0][3])) + flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction(lambda toks: ptypes.FlagsType(toks[0][0], toks[0][1], toks[0][2], toks[0][3])) + messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction(lambda toks: ptypes.MessageType(toks[0][1], toks[0][2], toks[0][3])) + channelDef = Group(channel_ + identifier + channelBody - semi).setParseAction(lambda toks: ptypes.ChannelType(toks[0][1], toks[0][2], toks[0][3])) + structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction(lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3])) + typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction(lambda toks: ptypes.TypeAlias(toks[0][1], toks[0][2], toks[0][3])) + + definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef + + protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi) \ + .setParseAction(lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2])) + protocolDef = Group(protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi) \ + .setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2])) + + bnf = ZeroOrMore (definitions) + protocolDef + StringEnd() + + singleLineComment = "//" + restOfLine + bnf.ignore( singleLineComment ) + bnf.ignore( cStyleComment ) + + return bnf + + +def parse(filename): + try: + bnf = SPICE_BNF() + types = bnf.parseFile(filename) + except ParseException, err: + print >> sys.stderr, err.line + print >> sys.stderr, " "*(err.column-1) + "^" + print >> sys.stderr, err + return None + + for t in types: + t.resolve() + t.register() + protocol = types[-1] + return protocol + diff --git a/spice.proto b/spice.proto new file mode 100644 index 0000000..dec6a63 --- /dev/null +++ b/spice.proto @@ -0,0 +1,1086 @@ +/* built in types: + int8, uint8, 16, 32, 64 +*/ + +typedef fixed28_4 int32 @ctype(SPICE_FIXED28_4); + +struct Point { + int32 x; + int32 y; +}; + +struct Point16 { + int16 x; + int16 y; +}; + +struct PointFix { + fixed28_4 x; + fixed28_4 y; +}; + +struct Rect { + int32 top; + int32 left; + int32 bottom; + int32 right; +}; + +enum32 link_err { + OK, + ERROR, + INVALID_MAGIC, + INVALID_DATA, + VERSION_MISMATCH, + NEED_SECURED, + NEED_UNSECURED, + PERMISSION_DENIED, + BAD_CONNECTION_ID, + CHANNEL_NOT_AVAILABLE +}; + +enum32 warn_code { + WARN_GENERAL +} @prefix(SPICE_); + +enum32 info_code { + INFO_GENERAL +} @prefix(SPICE_); + +flags32 migrate_flags { + NEED_FLUSH, + NEED_DATA_TRANSFER +} @prefix(SPICE_MIGRATE_); + +enum32 notify_severity { + INFO, + WARN, + ERROR, +}; + +enum32 notify_visibility { + LOW, + MEDIUM, + HIGH, +}; + +flags32 mouse_mode { + SERVER, + CLIENT, +}; + +enum16 pubkey_type { + INVALID, + RSA, + RSA2, + DSA, + DSA1, + DSA2, + DSA3, + DSA4, + DH, + EC, +}; + +message Empty { +}; + +message Data { + uint8 data[] @end @ctype(uint8_t); +} @nocopy; + +struct ChannelWait { + uint8 channel_type; + uint8 channel_id; + uint64 message_serial; +} @ctype(SpiceWaitForChannel); + +channel BaseChannel { + server: + message { + migrate_flags flags; + } migrate; + + Data migrate_data; + + message { + uint32 generation; + uint32 window; + } set_ack; + + message { + uint32 id; + uint64 timestamp; + uint8 data[] @end @ctype(uint8_t); + } ping; + + message { + uint8 wait_count; + ChannelWait wait_list[wait_count] @end; + } wait_for_channels; + + message { + uint64 time_stamp; + link_err reason; + } @ctype(SpiceMsgDisconnect) disconnecting; + + message { + uint64 time_stamp; + notify_severity severity; + notify_visibility visibilty; + uint32 what; /* error_code/warn_code/info_code */ + uint32 message_len; + uint8 message[message_len] @end; + uint8 zero @end @ctype(uint8_t) @zero; + } notify; + + client: + message { + uint32 generation; + } ack_sync; + + Empty ack; + + message { + uint32 id; + uint64 timestamp; + } @ctype(SpiceMsgPing) pong; + + Empty migrate_flush_mark; + + Data migrate_data; + + message { + uint64 time_stamp; + link_err reason; + } @ctype(SpiceMsgDisconnect) disconnecting; +}; + +struct ChannelId { + uint8 type; + uint8 id; +}; + +channel MainChannel : BaseChannel { + server: + message { + uint16 port; + uint16 sport; + uint32 host_offset; + uint32 host_size; + pubkey_type pub_key_type @minor(2); + uint32 pub_key_offset @minor(2); + uint32 pub_key_size @minor(2); + uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated; + uint8 pub_key_data[pub_key_size] @minor(2) @end @ctype(uint8_t) @zero_terminated; + } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; + + Empty migrate_cancel; + + message { + uint32 session_id; + uint32 display_channels_hint; + uint32 supported_mouse_modes; + uint32 current_mouse_mode; + uint32 agent_connected; + uint32 agent_tokens; + uint32 multi_media_time; + uint32 ram_hint; + } init; + + message { + uint32 num_of_channels; + ChannelId channels[num_of_channels] @end; + } @ctype(SpiceMsgChannels) channels_list; + + message { + mouse_mode supported_modes; + mouse_mode current_mode @unique_flag; + } mouse_mode; + + message { + uint32 time; + } @ctype(SpiceMsgMainMultiMediaTime) multi_media_time; + + Empty agent_connected; + + message { + link_err error_code; + } @ctype(SpiceMsgMainAgentDisconnect) agent_disconnected; + + Data agent_data; + + message { + uint32 num_tokens; + } @ctype(SpiceMsgMainAgentTokens) agent_token; + + message { + uint16 port; + uint16 sport; + uint32 host_offset; + uint32 host_size; + uint32 cert_subject_offset; + uint32 cert_subject_size; + uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated; + uint8 cert_subject_data[cert_subject_size] @end @ctype(uint8_t) @zero_terminated; + } @ctype(SpiceMsgMainMigrationSwitchHost) migrate_switch_host; + + client: + message { + uint64 cache_size; + } @ctype(SpiceMsgcClientInfo) client_info = 101; + + Empty migrate_connected; + + Empty migrate_connect_error; + + Empty attach_channels; + + message { + mouse_mode mode; + } mouse_mode_request; + + message { + uint32 num_tokens; + } agent_start; + + Data agent_data; + + message { + uint32 num_tokens; + } @ctype(SpiceMsgcMainAgentTokens) agent_token; +}; + +enum32 clip_type { + NONE, + RECTS, + PATH, +}; + +flags32 path_flags { /* TODO: C enum names changes */ + BEGIN = 0, + END = 1, + CLOSE = 3, + BEZIER = 4, +} @prefix(SPICE_PATH_); + +enum32 video_codec_type { + MJPEG = 1, +}; + +flags32 stream_flags { + TOP_DOWN = 0, +}; + +enum32 brush_type { + NONE, + SOLID, + PATTERN, +}; + +flags8 mask_flags { + INVERS, +}; + +enum8 image_type { + BITMAP, + QUIC, + RESERVED, + LZ_PLT = 100, + LZ_RGB, + GLZ_RGB, + FROM_CACHE, + SURFACE, + JPEG, + FROM_CACHE_LOSSLESS, +}; + +flags8 image_flags { + CACHE_ME, + HIGH_BITS_SET, + CACHE_REPLACE_ME, +}; + +enum8 bitmap_fmt { + INVALID, + 1BIT_LE, + 1BIT_BE, + 4BIT_LE, + 4BIT_BE, + 8BIT /* 8bit indexed mode */, + 16BIT, /* 0555 mode */ + 24BIT /* 3 byte, brg */, + 32BIT /* 4 byte, xrgb in little endian format */, + RGBA /* 4 byte, argb in little endian format */ +}; + +flags8 bitmap_flags { + PAL_CACHE_ME, + PAL_FROM_CACHE, + TOP_DOWN, +}; + +enum8 image_scale_mode { + INTERPOLATE, + NEAREST, +}; + +flags16 ropd { + INVERS_SRC, + INVERS_BRUSH, + INVERS_DEST, + OP_PUT, + OP_OR, + OP_AND, + OP_XOR, + OP_BLACKNESS, + OP_WHITENESS, + OP_INVERS, + INVERS_RES, +}; + +flags8 line_flags { + STYLED = 3, + START_WITH_GAP = 2, +}; + +enum8 line_cap { + ROUND, + SQUARE, + BUTT, +}; + +enum8 line_join { + ROUND, + BEVEL, + MITER, +}; + +flags16 string_flags { + RASTER_A1, + RASTER_A4, + RASTER_A8, + RASTER_TOP_DOWN, +}; + +flags32 surface_flags { + PRIMARY +}; + +enum32 surface_fmt { + INVALID, + 1_A = 1, + 8_A = 8, + 16_555 = 16 , + 16_565 = 80, + 32_xRGB = 32, + 32_ARGB = 96 +}; + +flags16 alpha_flags { + DEST_HAS_ALPHA, + SRC_SURFACE_HAS_ALPHA +}; + +enum8 resource_type { + INVALID, + PIXMAP +} @prefix(SPICE_RES_TYPE_); + +struct ClipRects { + uint32 num_rects; + Rect rects[num_rects] @end; +}; + +struct PathSegment { + path_flags flags; + uint32 count; + PointFix points[count] @end; +} @ctype(SpicePathSeg); + +struct Path { + uint32 size; + PathSegment segments[bytes(size)] @end; +}; + +struct Clip { + clip_type type; + switch (type) { + case NONE: + uint64 data @zero; + case RECTS: + ClipRects *data; + case PATH: + Path *data; + } u @anon; +}; + +struct DisplayBase { + uint32 surface_id; + Rect box; + Clip clip; +} @ctype(SpiceMsgDisplayBase); + +struct ResourceID { + uint8 type; + uint64 id; +}; + +struct WaitForChannel { + uint8 channel_type; + uint8 channel_id; + uint64 message_serial; +}; + +struct Palette { + uint64 unique; + uint16 num_ents; + uint32 ents[num_ents] @end; +}; + +struct BitmapData { + bitmap_fmt format; + bitmap_flags flags; + uint32 x; + uint32 y; + uint32 stride; + switch (flags) { + case PAL_FROM_CACHE: + uint64 palette; + default: + Palette *palette; + } pal @anon; + uint8 *data[image_size(8, stride, y)] @nocopy; /* pointer to array, not array of pointers as in C */ +} @ctype(SpiceBitmap); + +struct BinaryData { + uint32 data_size; + uint8 data[data_size] @end; +} @ctype(SpiceQUICData); + +struct LZPLTData { + bitmap_flags flags; + uint32 data_size; + switch (flags) { + case PAL_FROM_CACHE: + uint64 palette; + default: + Palette *palette @nonnull; + } pal @anon; + uint8 data[data_size] @end; +}; + +struct Surface { + uint32 surface_id; +}; + +struct Image { + uint64 id; + image_type type; + image_flags flags; + uint32 width; + uint32 height; + + switch (type) { + case BITMAP: + BitmapData bitmap_data @ctype(SpiceBitmap); + case QUIC: + case LZ_RGB: + case GLZ_RGB: + case JPEG: + BinaryData binary_data @ctype(SpiceQUICData); + case LZ_PLT: + LZPLTData lzplt_data @ctype(SpiceLZPLTData); + case SURFACE: + Surface surface_data; + } u @end; +} @ctype(SpiceImageDescriptor); + +struct Pattern { + Image *pat @nonnull; + Point pos; +}; + +struct Brush { + brush_type type; + switch (type) { + case SOLID: + uint32 color; + case PATTERN: + Pattern pattern; + } u @fixedsize; +}; + +struct QMask { + mask_flags flags; + Point pos; + Image *bitmap; +}; + +struct LineAttr { + line_flags flags; + line_join join_style; + line_cap end_style; + uint8 style_nseg; + fixed28_4 width; + fixed28_4 miter_limit; + fixed28_4 *style[style_nseg]; +}; + +struct RasterGlyphA1 { + Point render_pos; + Point glyph_origin; + uint16 width; + uint16 height; + uint8 data[image_size(1, width, height)] @end; +} @ctype(SpiceRasterGlyph); + +struct RasterGlyphA4 { + Point render_pos; + Point glyph_origin; + uint16 width; + uint16 height; + uint8 data[image_size(4, width, height)] @end; +} @ctype(SpiceRasterGlyph); + +struct RasterGlyphA8 { + Point render_pos; + Point glyph_origin; + uint16 width; + uint16 height; + uint8 data[image_size(8, width, height)] @end; +} @ctype(SpiceRasterGlyph); + +struct String { + uint16 length; + string_flags flags; /* Special: Only one of a1/a4/a8 set */ + switch (flags) { + case RASTER_A1: + RasterGlyphA1 glyphs[length] @ctype(SpiceRasterGlyph); + case RASTER_A4: + RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph); + case RASTER_A8: + RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph); + } u @end; +}; + +channel DisplayChannel : BaseChannel { + server: + message { + uint32 x_res; + uint32 y_res; + uint32 bits; + } mode = 101; + + Empty mark; + Empty reset; + message { + DisplayBase base; + Point src_pos; + } copy_bits; + + message { + uint16 count; + ResourceID resources[count] @end; + } @ctype(SpiceResourceList) inval_list; + + message { + uint8 wait_count; + WaitForChannel wait_list[wait_count] @end; + } @ctype(SpiceMsgWaitForChannels) inval_all_pixmaps; + + message { + uint64 id; + } @ctype(SpiceMsgDisplayInvalOne) inval_palette; + + Empty inval_all_palettes; + + message { + uint32 surface_id; + uint32 id; + stream_flags flags; + video_codec_type codec_type; + uint64 stamp; + uint32 stream_width; + uint32 stream_height; + uint32 src_width; + uint32 src_height; + Rect dest; + Clip clip; + } stream_create = 122; + + message { + uint32 id; + uint32 multi_media_time; + uint32 data_size; + uint32 pad_size; + uint8 data[data_size] @end; + uint8 padding[pad_size] @end @ctype(uint8_t); /* Uhm, why are we sending padding over network? */ + } stream_data; + + message { + uint32 id; + Clip clip; + } stream_clip; + + message { + uint32 id; + } stream_destroy; + + Empty stream_destroy_all; + + message { + DisplayBase base; + struct Fill { + Brush brush; + uint16 rop_decriptor; + QMask mask; + } data; + } draw_fill = 302; + + message { + DisplayBase base; + struct Opaque { + Image *src_bitmap; + Rect src_area; + Brush brush; + ropd rop_decriptor; + image_scale_mode scale_mode; + QMask mask; + } data; + } draw_opaque; + + message { + DisplayBase base; + struct Copy { + Image *src_bitmap; + Rect src_area; + ropd rop_decriptor; + image_scale_mode scale_mode; + QMask mask; + } data; + } draw_copy; + + message { + DisplayBase base; + struct Blend { + Image *src_bitmap; + Rect src_area; + ropd rop_decriptor; + image_scale_mode scale_mode; + QMask mask; + } @ctype(SpiceCopy) data; + } draw_blend; + + message { + DisplayBase base; + struct Blackness { + QMask mask; + } data; + } draw_blackness; + + message { + DisplayBase base; + struct Whiteness { + QMask mask; + } data; + } draw_whiteness; + + message { + DisplayBase base; + struct Invers { + QMask mask; + } data; + } draw_invers; + + message { + DisplayBase base; + struct Rop3 { + Image *src_bitmap; + Rect src_area; + Brush brush; + uint8 rop3; + image_scale_mode scale_mode; + QMask mask; + } data; + } draw_rop3; + + message { + DisplayBase base; + struct Stroke { + Path *path; + LineAttr attr; + Brush brush; + uint16 fore_mode; + uint16 back_mode; + } data; + } draw_stroke; + + message { + DisplayBase base; + struct Text { + String *str; + Rect back_area; + Brush fore_brush; + Brush back_brush; + uint16 fore_mode; + uint16 back_mode; + } data; + } draw_text; + + message { + DisplayBase base; + struct Transparent { + Image *src_bitmap; + Rect src_area; + uint32 src_color; + uint32 true_color; + } data; + } draw_transparent; + + message { + DisplayBase base; + struct AlphaBlnd { + alpha_flags alpha_flags; + uint8 alpha; + Image *src_bitmap; + Rect src_area; + } data; + } draw_alpha_blend; + + message { + uint32 surface_id; + uint32 width; + uint32 height; + uint32 format; + surface_flags flags; + } @ctype(SpiceMsgSurfaceCreate) surface_create; + + message { + uint32 surface_id; + } @ctype(SpiceMsgSurfaceDestroy) surface_destroy; + + client: + message { + uint8 pixmap_cache_id; + int64 pixmap_cache_size; //in pixels + uint8 glz_dictionary_id; + int32 glz_dictionary_window_size; // in pixels + } init = 101; +}; + +flags32 keyboard_modifier_flags { + SCROLL_LOCK, + NUM_LOCK, + CAPS_LOCK +}; + +enum32 mouse_button { + INVALID, + LEFT, + MIDDLE, + RIGHT, + UP, + DOWN, +}; + +flags32 mouse_button_mask { + LEFT, + MIDDLE, + RIGHT +}; + +channel InputsChannel : BaseChannel { + client: + message { + uint32 code; + } @ctype(SpiceMsgcKeyDown) key_down = 101; + + message { + uint32 code; + } @ctype(SpiceMsgcKeyUp) key_up; + + message { + keyboard_modifier_flags modifiers; + } @ctype(SpiceMsgcKeyModifiers) key_modifiers; + + message { + int32 dx; + int32 dy; + mouse_button_mask buttons_state; + } @ctype(SpiceMsgcMouseMotion) mouse_motion = 111; + + message { + uint32 x; + uint32 y; + mouse_button_mask buttons_state; + uint8 display_id; + } @ctype(SpiceMsgcMousePosition) mouse_position; + + message { + mouse_button button; + mouse_button_mask buttons_state; + } @ctype(SpiceMsgcMousePress) mouse_press; + + message { + mouse_button button; + mouse_button_mask buttons_state; + } @ctype(SpiceMsgcMouseRelease) mouse_release; + + server: + message { + keyboard_modifier_flags keyboard_modifiers; + } init = 101; + + message { + keyboard_modifier_flags modifiers; + } key_modifiers; + + Empty mouse_motion_ack = 111; +}; + +enum16 cursor_type { + ALPHA, + MONO, + COLOR4, + COLOR8, + COLOR16, + COLOR24, + COLOR32, +}; + +flags32 cursor_flags { + NONE, /* Means no cursor */ + CACHE_ME, + FROM_CACHE, +}; + +struct CursorHeader { + uint64 unique; + cursor_type type; + uint16 width; + uint16 height; + uint16 hot_spot_x; + uint16 hot_spot_y; +}; + +struct Cursor { + cursor_flags flags; + CursorHeader header; + uint8 data[] @end; +}; + +channel CursorChannel : BaseChannel { + server: + message { + Point16 position; + uint16 trail_length; + uint16 trail_frequency; + uint8 visible; + Cursor cursor; + } init = 101; + + Empty reset; + + message { + Point16 position; + uint8 visible; + Cursor cursor; + } set; + + message { + Point16 position; + } move; + + Empty hide; + + message { + uint16 length; + uint16 frequency; + } trail; + + message { + uint64 id; + } @ctype(SpiceMsgDisplayInvalOne) inval_one; + + Empty inval_all; +}; + +enum32 audio_data_mode { + INVALID, + RAW, + CELT_0_5_1, +}; + +enum32 audio_fmt { + INVALID, + S16, +}; + +channel PlaybackChannel : BaseChannel { + server: + message { + uint32 time; + uint8 data[] @end; + } @ctype(SpiceMsgPlaybackPacket) data = 101; + + message { + uint32 time; + audio_data_mode mode; + uint8 data[] @end; + } mode; + + message { + uint32 channels; + audio_fmt format; + uint32 frequency; + uint32 time; + } start; + + Empty stop; +}; + +channel RecordChannel : BaseChannel { + server: + message { + uint32 channels; + audio_fmt format; + uint32 frequency; + } start = 101; + + Empty stop; + client: + message { + uint32 time; + uint8 data[] @end; + } @ctype(SpiceMsgcRecordPacket) data = 101; + + message { + uint32 time; + audio_data_mode mode; + uint8 data[] @end; + } mode; + + message { + uint32 time; + } start_mark; +}; + +enum32 tunnel_service_type { + INVALID, + GENERIC, + IPP, +}; + +enum16 tunnel_ip_type { + INVALID, + IPv4, +}; + +struct TunnelIpInfo { + tunnel_ip_type type; + switch (type) { + case IPv4: + uint8 ipv4[4] @ctype(uint8_t); + } u @end; +} @ctype(SpiceMsgTunnelIpInfo); + +channel TunnelChannel : BaseChannel { + server: + message { + uint16 max_num_of_sockets; + uint32 max_socket_data_size; + } init = 101; + + message { + uint32 service_id; + TunnelIpInfo virtual_ip; + } service_ip_map; + + message { + uint16 connection_id; + uint32 service_id; + uint32 tokens; + } socket_open; + + message { + uint16 connection_id; + } socket_fin; + + message { + uint16 connection_id; + } socket_close; + + message { + uint16 connection_id; + uint8 data[] @end; + } socket_data; + + message { + uint16 connection_id; + } socket_closed_ack; + + message { + uint16 connection_id; + uint32 num_tokens; + } @ctype(SpiceMsgTunnelSocketTokens) socket_token; + + client: + message { + tunnel_service_type type; + uint32 id; + uint32 group; + uint32 port; + uint32 name; + uint32 description; + switch (type) { + case IPP: + TunnelIpInfo ip @ctype(SpiceMsgTunnelIpInfo); + } u @end; + } @ctype(SpiceMsgcTunnelAddGenericService) service_add = 101; + + message { + uint32 id; + } @ctype(SpiceMsgcTunnelRemoveService) service_remove; + + message { + uint16 connection_id; + uint32 tokens; + } socket_open_ack; + + message { + uint16 connection_id; + } socket_open_nack; + + message { + uint16 connection_id; + } socket_fin; + + message { + uint16 connection_id; + } socket_closed; + + message { + uint16 connection_id; + } socket_closed_ack; + + message { + uint16 connection_id; + uint8 data[] @end; + } socket_data; + + message { + uint16 connection_id; + uint32 num_tokens; + } @ctype(SpiceMsgcTunnelSocketTokens) socket_token; +}; + +protocol Spice { + MainChannel main = 1; + DisplayChannel display; + InputsChannel inputs; + CursorChannel cursor; + PlaybackChannel playback; + RecordChannel record; + TunnelChannel tunnel; +}; From 809524ac5debf69ffc8ae5498a6fc305c51fc4be Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 26 May 2010 12:24:48 +0200 Subject: [PATCH 002/147] Add python code to automake system --- python_modules/Makefile.am | 6 ++ spice_codegen.py | 165 +++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 python_modules/Makefile.am create mode 100755 spice_codegen.py diff --git a/python_modules/Makefile.am b/python_modules/Makefile.am new file mode 100644 index 0000000..4b3c960 --- /dev/null +++ b/python_modules/Makefile.am @@ -0,0 +1,6 @@ +NULL = + +PYTHON_MODULES = __init__.py codegen.py demarshal.py ptypes.py spice_parser.py + +EXTRA_DIST = $(PYTHON_MODULES) + diff --git a/spice_codegen.py b/spice_codegen.py new file mode 100755 index 0000000..f897ce8 --- /dev/null +++ b/spice_codegen.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python + +import os +import sys +from optparse import OptionParser +import traceback +from python_modules import spice_parser +from python_modules import ptypes +from python_modules import codegen +from python_modules import demarshal + +def write_channel_enums(writer, channel, client): + messages = filter(lambda m : m.channel == channel, \ + channel.client_messages if client else channel.server_messages) + if len(messages) == 0: + return + writer.begin_block("enum") + i = 0; + if client: + prefix = [ "MSGC" ] + else: + prefix = [ "MSG" ] + if channel.member_name: + prefix.append(channel.member_name.upper()) + prefix.append(None) # To be replaced with name + for m in messages: + prefix[-1] = m.name.upper() + enum = codegen.prefix_underscore_upper(*prefix) + if m.value == i: + writer.writeln("%s," % enum) + i = i + 1 + else: + writer.writeln("%s = %s," % (enum, m.value)) + i = m.value + 1 + if channel.member_name: + prefix[-1] = prefix[-2] + prefix[-2] = "END" + writer.newline() + writer.writeln("%s" % (codegen.prefix_underscore_upper(*prefix))) + writer.end_block(semicolon=True) + writer.newline() + +def write_enums(writer): + writer.writeln("#ifndef _H_SPICE_ENUMS") + writer.writeln("#define _H_SPICE_ENUMS") + writer.newline() + writer.comment("Generated from %s, don't edit" % writer.options["source"]).newline() + writer.newline() + + # Define enums + for t in ptypes.get_named_types(): + if isinstance(t, ptypes.EnumBaseType): + t.c_define(writer) + + i = 0; + writer.begin_block("enum") + for c in proto.channels: + enum = codegen.prefix_underscore_upper("CHANNEL", c.name.upper()) + if c.value == i: + writer.writeln("%s," % enum) + i = i + 1 + else: + writer.writeln("%s = %s," % (enum, c.value)) + i = c.value + 1 + writer.newline() + writer.writeln("SPICE_END_CHANNEL") + writer.end_block(semicolon=True) + writer.newline() + + for c in ptypes.get_named_types(): + if not isinstance(c, ptypes.ChannelType): + continue + write_channel_enums(writer, c, False) + write_channel_enums(writer, c, True) + + writer.writeln("#endif /* _H_SPICE_ENUMS */") + +parser = OptionParser(usage="usage: %prog [options] ") +parser.add_option("-e", "--generate-enums", + action="store_true", dest="generate_enums", default=False, + help="Generate enums") +parser.add_option("-d", "--generate-demarshallers", + action="store_true", dest="generate_demarshallers", default=False, + help="Generate demarshallers") +parser.add_option("-a", "--assert-on-error", + action="store_true", dest="assert_on_error", default=False, + help="Assert on error") +parser.add_option("-p", "--print-error", + action="store_true", dest="print_error", default=False, + help="Print errors") +parser.add_option("-s", "--server", + action="store_true", dest="server", default=False, + help="Print errors") +parser.add_option("-c", "--client", + action="store_true", dest="client", default=False, + help="Print errors") +parser.add_option("-k", "--keep-identical-file", + action="store_true", dest="keep_identical_file", default=False, + help="Print errors") +parser.add_option("-i", "--include", + dest="include", default=None, metavar="FILE", + help="Include FILE in generated code") + +(options, args) = parser.parse_args() + +if len(args) == 0: + parser.error("No protocol file specified") + +if len(args) == 1: + parser.error("No destination file specified") + +proto_file = args[0] +dest_file = args[1] +proto = spice_parser.parse(proto_file) + +if proto == None: + exit(1) + +codegen.set_prefix(proto.name) +writer = codegen.CodeWriter() +writer.set_option("source", os.path.basename(proto_file)) + +if options.assert_on_error: + writer.set_option("assert_on_error") + +if options.print_error: + writer.set_option("print_error") + +if options.include: + writer.writeln('#include "%s"' % options.include) + +if options.generate_enums: + write_enums(writer) + +if options.generate_demarshallers: + if not options.server and not options.client: + print >> sys.stderr, "Must specify client and/or server" + sys.exit(1) + demarshal.write_includes(writer) + + if options.server: + demarshal.write_protocol_parser(writer, proto, False) + if options.client: + demarshal.write_protocol_parser(writer, proto, True) + +content = writer.getvalue() +if options.keep_identical_file: + try: + f = open(dest_file, 'rb') + old_content = f.read() + f.close() + + if content == old_content: + print "No changes to %s" % dest_file + sys.exit(0) + + except IOError: + pass + +f = open(dest_file, 'wb') +f.write(content) +f.close() + +print "Wrote %s" % dest_file +sys.exit(0) From 79bd3284c63db1a88c1f05ba6d867bb13ba82a03 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 14 Jun 2010 16:11:39 +0200 Subject: [PATCH 003/147] Add support for generating message and structure marshallers --- python_modules/marshal.py | 357 ++++++++++++++++++++++++++++++++++++++ spice_codegen.py | 36 +++- 2 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 python_modules/marshal.py diff --git a/python_modules/marshal.py b/python_modules/marshal.py new file mode 100644 index 0000000..23b029a --- /dev/null +++ b/python_modules/marshal.py @@ -0,0 +1,357 @@ +import ptypes +import codegen + +def write_includes(writer): + writer.header.writeln("#include ") + writer.header.writeln("#include ") + writer.header.newline() + writer.header.writeln("#ifndef _GENERATED_HEADERS_H") + writer.header.writeln("#define _GENERATED_HEADERS_H") + + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.writeln("#include ") + writer.newline() + writer.writeln("#ifdef _MSC_VER") + writer.writeln("#pragma warning(disable:4101)") + writer.writeln("#endif") + writer.newline() + +class MarshallingSource: + def __init__(self): + pass + + def child_at_end(self, t): + return RootMarshallingSource(self, t.c_type(), t.sizeof()) + + def child_sub(self, member): + return SubMarshallingSource(self, member) + + def declare(self, writer): + return writer.optional_block(self.reuse_scope) + + def is_toplevel(self): + return self.parent_src == None and not self.is_helper + +class RootMarshallingSource(MarshallingSource): + def __init__(self, parent_src, c_type, sizeof, pointer = None): + self.is_helper = False + self.reuse_scope = None + self.parent_src = parent_src + if parent_src: + self.base_var = codegen.increment_identifier(parent_src.base_var) + else: + self.base_var = "src" + self.c_type = c_type + self.sizeof = sizeof + self.pointer = pointer # None == at "end" + + def get_self_ref(self): + return self.base_var + + def get_ref(self, member): + return self.base_var + "->" + member + + def declare(self, writer): + if self.reuse_scope: + scope = self.reuse_scope + else: + writer.begin_block() + scope = writer.get_subwriter() + + scope.variable_def(self.c_type + " *", self.base_var) + if not self.reuse_scope: + scope.newline() + + if self.pointer: + writer.assign(self.base_var, "(%s *)%s" % (self.c_type, self.pointer)) + else: + writer.assign(self.base_var, "(%s *)end" % self.c_type) + writer.increment("end", "%s" % self.sizeof) + writer.newline() + + if self.reuse_scope: + return writer.no_block(self.reuse_scope) + else: + return writer.partial_block(scope) + +class SubMarshallingSource(MarshallingSource): + def __init__(self, parent_src, member): + self.reuse_scope = None + self.parent_src = parent_src + self.base_var = parent_src.base_var + self.member = member + self.is_helper = False + + def get_self_ref(self): + return "&%s" % self.parent_src.get_ref(self.member) + + def get_ref(self, member): + return self.parent_src.get_ref(self.member) + "." + member + +def write_marshal_ptr_function(writer, target_type): + if target_type.is_array(): + marshal_function = "spice_marshall_array_%s" % target_type.element_type.primitive_type() + else: + marshal_function = "spice_marshall_%s" % target_type.name + if writer.is_generated("marshaller", marshal_function): + return marshal_function + + writer.set_is_generated("marshaller", marshal_function) + + names = target_type.get_pointer_names() + names_args = "" + if len(names) > 0: + n = map(lambda name: ", SpiceMarshaller **%s" % name, names) + names_args = "".join(n) + + header = writer.header + writer = writer.function_helper() + writer.header = header + writer.out_prefix = "" + if target_type.is_array(): + scope = writer.function(marshal_function, "void *", "SpiceMarshaller *m, %s_t *ptr, int count" % target_type.element_type.primitive_type() + names_args) + else: + scope = writer.function(marshal_function, "void *", "SpiceMarshaller *m, %s *ptr" % target_type.c_type() + names_args) + header.writeln("void *" + marshal_function + "(SpiceMarshaller *m, %s *msg" % target_type.c_type() + names_args + ");") + scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") + + for n in names: + writer.assign("*%s" % n, "NULL") + + writer.newline() + writer.assign("end", "(uint8_t *)(ptr+1)") + + if target_type.is_struct(): + src = RootMarshallingSource(None, target_type.c_type(), target_type.sizeof(), "ptr") + src.reuse_scope = scope + write_container_marshaller(writer, target_type, src) + elif target_type.is_array() and target_type.element_type.is_primitive(): + with writer.index() as index: + with writer.for_loop(index, "count") as array_scope: + writer.statement("spice_marshaller_add_%s(m, *ptr++)" % (target_type.element_type.primitive_type())) + else: + writer.todo("Unsuppored pointer marshaller type") + + writer.statement("return end") + + writer.end_block() + + return marshal_function + +def get_array_size(array, container_src): + if array.is_constant_length(): + return array.size + elif array.is_identifier_length(): + return container_src.get_ref(array.size) + elif array.is_remaining_length(): + raise NotImplementedError("remaining size array sizes marshalling not supported") + elif array.is_image_size_length(): + bpp = array.size[1] + width = array.size[2] + rows = array.size[3] + width_v = container_src.get_ref(width) + rows_v = container_src.get_ref(rows) + # TODO: Handle multiplication overflow + if bpp == 8: + return "(%s * %s)" % (width_v, rows_v) + elif bpp == 1: + return "(((%s + 7) / 8 ) * %s)" % (width_v, rows_v) + else: + return "(((%s * %s + 7) / 8 ) * %s)" % (bpp, width_v, rows_v) + elif array.is_bytes_length(): + return container_src.get_ref(array.size[1]) + else: + raise NotImplementedError("TODO array size type not handled yet") + +def write_array_marshaller(writer, at_end, member, array, container_src, scope): + element_type = array.element_type + + if array.is_remaining_length(): + writer.comment("Remaining data must be appended manually").newline() + return + + nelements = get_array_size(array, container_src) + is_byte_size = array.is_bytes_length() + + if is_byte_size: + element = "%s__bytes" % member.name + else: + element = "%s__element" % member.name + + if not at_end: + writer.assign(element, container_src.get_ref(member.name)) + + if is_byte_size: + scope.variable_def("size_t", "array_end") + writer.assign("array_end", "spice_marshaller_get_size(m) + %s" % nelements) + + with writer.index(no_block = is_byte_size) as index: + with writer.while_loop("spice_marshaller_get_size(m) < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: + array_scope.variable_def(element_type.c_type() + " *", element) + if at_end: + writer.assign(element, "(%s *)end" % element_type.c_type()) + writer.increment("end", element_type.sizeof()) + + if element_type.is_primitive(): + writer.statement("spice_marshaller_add_%s(m, *%s)" % (element_type.primitive_type(), element)) + elif element_type.is_struct(): + src2 = RootMarshallingSource(container_src, element_type.c_type(), element_type.sizeof(), element) + src2.reuse_scope = array_scope + write_container_marshaller(writer, element_type, src2) + else: + writer.todo("array element unhandled type").newline() + + if not at_end: + writer.statement("%s++" % element) + +def write_switch_marshaller(writer, container, switch, src, scope): + var = container.lookup_member(switch.variable) + var_type = var.member_type + + saved_out_prefix = writer.out_prefix + first = True + for c in switch.cases: + check = c.get_check(src.get_ref(switch.variable), var_type) + m = c.member + writer.out_prefix = saved_out_prefix + if m.has_attr("outvar"): + writer.out_prefix = "%s_%s" % (m.attributes["outvar"][0], writer.out_prefix) + with writer.if_block(check, not first, False) as block: + t = m.member_type + if switch.has_end_attr(): + src2 = src.child_at_end(m.member_type) + elif switch.has_attr("anon"): + src2 = src + else: + if t.is_struct(): + src2 = src.child_sub(switch.name + "." + m.name) + else: + src2 = src.child_sub(switch.name) + src2.reuse_scope = block + + if t.is_struct(): + write_container_marshaller(writer, t, src2) + elif t.is_pointer(): + ptr_func = write_marshal_ptr_function(writer, t.target_type) + writer.assign("*%s_out" % (writer.out_prefix + m.name), "spice_marshaller_get_ptr_submarshaller(m, %s)" % ("0" if m.has_attr("ptr32") else "1")) + elif t.is_primitive(): + if m.has_attr("zero"): + writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) + else: + writer.statement("spice_marshaller_add_%s(m, %s)" % (t.primitive_type(), src2.get_ref(m.name))) + #TODO validate e.g. flags and enums + elif t.is_array(): + write_array_marshaller(writer, switch.has_end_attr(), m, t, src, scope) + else: + writer.todo("Can't handle type %s" % m.member_type) + + if switch.has_attr("fixedsize"): + remaining = switch.get_fixed_nw_size() - t.get_fixed_nw_size() + if remaining != 0: + writer.statement("spice_marshaller_reserve_space(m, %s)" % remaining) + + first = False + if switch.has_attr("fixedsize"): + with writer.block(" else"): + writer.statement("spice_marshaller_reserve_space(m, %s)" % switch.get_fixed_nw_size()) + + writer.newline() + +def write_member_marshaller(writer, container, member, src, scope): + if member.has_attr("outvar"): + writer.out_prefix = "%s_%s" % (member.attributes["outvar"][0], writer.out_prefix) + if member.has_attr("nomarshal"): + writer.comment("Don't marshall @nomarshal %s" % member.name).newline() + return + if member.is_switch(): + write_switch_marshaller(writer, container, member, src, scope) + return + + t = member.member_type + + if t.is_pointer(): +# if member.has_attr("nocopy"): +# writer.comment("Reuse data from network message").newline() +# writer.assign(src.get_ref(member.name), "(size_t)(message_start + consume_uint64(&in))") +# else: +# write_parse_pointer(writer, t, member.has_end_attr(), src, member.name, scope) + ptr_func = write_marshal_ptr_function(writer, t.target_type) + writer.assign("*%s_out" % (writer.out_prefix + member.name), "spice_marshaller_get_ptr_submarshaller(m, %s)" % ("0" if member.has_attr("ptr32") else "1")) + elif t.is_primitive(): + if member.has_attr("zero"): + writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) + elif member.has_end_attr(): + writer.statement("spice_marshaller_add_%s(m, *(%s_t *)end)" % (t.primitive_type(), t.primitive_type())) + writer.increment("end", t.sizeof()) + else: + writer.statement("spice_marshaller_add_%s(m, %s)" % (t.primitive_type(), src.get_ref(member.name))) + elif t.is_array(): + write_array_marshaller(writer, member.has_end_attr(), member, t, src, scope) + elif t.is_struct(): + if member.has_end_attr(): + src2 = src.child_at_end(t) + else: + src2 = src.child_sub(member.name) + writer.comment(member.name) + write_container_marshaller(writer, t, src2) + else: + raise NotImplementedError("TODO can't handle parsing of %s" % t) + +def write_container_marshaller(writer, container, src): + saved_out_prefix = writer.out_prefix + with src.declare(writer) as scope: + for m in container.members: + writer.out_prefix = saved_out_prefix + write_member_marshaller(writer, container, m, src, scope) + +def write_message_marshaller(writer, message, is_server): + writer.out_prefix = "" + function_name = "spice_marshall_" + message.c_name() + if writer.is_generated("marshaller", function_name): + return function_name + writer.set_is_generated("marshaller", function_name) + + names = message.get_pointer_names() + names_args = "" + if len(names) > 0: + n = map(lambda name: ", SpiceMarshaller **%s" % name, names) + names_args = "".join(n) + + writer.header.writeln("void " + function_name + "(SpiceMarshaller *m, %s *msg" % message.c_type() + names_args + ");") + + scope = writer.function(function_name, + "void", + "SpiceMarshaller *m, %s *msg" % message.c_type() + names_args) + scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") + + for n in names: + writer.assign("*%s" % n, "NULL") + + src = RootMarshallingSource(None, message.c_type(), message.sizeof(), "msg") + src.reuse_scope = scope + + writer.assign("end", "(uint8_t *)(msg+1)") + write_container_marshaller(writer, message, src) + + writer.end_block() + writer.newline() + +def write_protocol_marshaller(writer, proto, is_server): + for c in proto.channels: + channel = c.channel_type + if is_server: + for m in channel.client_messages: + message = m.message_type + write_message_marshaller(writer, message, is_server) + else: + for m in channel.server_messages: + message = m.message_type + write_message_marshaller(writer, message, is_server) + +def write_trailer(writer): + writer.header.writeln("#endif") diff --git a/spice_codegen.py b/spice_codegen.py index f897ce8..0e9551f 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -8,6 +8,7 @@ from python_modules import spice_parser from python_modules import ptypes from python_modules import codegen from python_modules import demarshal +from python_modules import marshal def write_channel_enums(writer, channel, client): messages = filter(lambda m : m.channel == channel, \ @@ -82,9 +83,18 @@ parser.add_option("-e", "--generate-enums", parser.add_option("-d", "--generate-demarshallers", action="store_true", dest="generate_demarshallers", default=False, help="Generate demarshallers") +parser.add_option("-m", "--generate-marshallers", + action="store_true", dest="generate_marshallers", default=False, + help="Generate message marshallers") +parser.add_option("-M", "--generate-struct-marshaller", + action="append", dest="struct_marshallers", + help="Generate struct marshallers") parser.add_option("-a", "--assert-on-error", action="store_true", dest="assert_on_error", default=False, help="Assert on error") +parser.add_option("-H", "--header", + action="store_true", dest="header", default=False, + help="Generate header") parser.add_option("-p", "--print-error", action="store_true", dest="print_error", default=False, help="Print errors") @@ -118,6 +128,7 @@ if proto == None: codegen.set_prefix(proto.name) writer = codegen.CodeWriter() +writer.header = codegen.CodeWriter() writer.set_option("source", os.path.basename(proto_file)) if options.assert_on_error: @@ -143,7 +154,30 @@ if options.generate_demarshallers: if options.client: demarshal.write_protocol_parser(writer, proto, True) -content = writer.getvalue() +if options.generate_marshallers or (options.struct_marshallers and len(options.struct_marshallers) > 0): + marshal.write_includes(writer) + +if options.generate_marshallers: + if not options.server and not options.client: + print >> sys.stderr, "Must specify client and/or server" + sys.exit(1) + if options.server: + marshal.write_protocol_marshaller(writer, proto, False) + if options.client: + marshal.write_protocol_marshaller(writer, proto, True) + +if options.struct_marshallers: + for structname in options.struct_marshallers: + t = ptypes.lookup_type(structname) + marshal.write_marshal_ptr_function(writer, t) + +if options.generate_marshallers or (options.struct_marshallers and len(options.struct_marshallers) > 0): + marshal.write_trailer(writer) + +if options.header: + content = writer.header.getvalue() +else: + content = writer.getvalue() if options.keep_identical_file: try: f = open(dest_file, 'rb') From a4cb48f1baa3d1a1379bf973965aac0b2b52cbf5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 1 Jun 2010 21:54:16 +0200 Subject: [PATCH 004/147] spice.proto: add @outvar markup With this we can reference pointer member with no naming conflicts --- spice.proto | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/spice.proto b/spice.proto index dec6a63..32ad869 100644 --- a/spice.proto +++ b/spice.proto @@ -409,9 +409,9 @@ struct Clip { case NONE: uint64 data @zero; case RECTS: - ClipRects *data; + ClipRects *data @outvar(cliprects); case PATH: - Path *data; + Path *data @outvar(clippath); } u @anon; }; @@ -448,7 +448,7 @@ struct BitmapData { case PAL_FROM_CACHE: uint64 palette; default: - Palette *palette; + Palette *palette @outvar(bitmap); } pal @anon; uint8 *data[image_size(8, stride, y)] @nocopy; /* pointer to array, not array of pointers as in C */ } @ctype(SpiceBitmap); @@ -465,7 +465,7 @@ struct LZPLTData { case PAL_FROM_CACHE: uint64 palette; default: - Palette *palette @nonnull; + Palette *palette @nonnull @outvar(lzplt); } pal @anon; uint8 data[data_size] @end; }; @@ -632,9 +632,9 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Fill { - Brush brush; + Brush brush @outvar(brush); uint16 rop_decriptor; - QMask mask; + QMask mask @outvar(mask); } data; } draw_fill = 302; @@ -646,7 +646,7 @@ channel DisplayChannel : BaseChannel { Brush brush; ropd rop_decriptor; image_scale_mode scale_mode; - QMask mask; + QMask mask @outvar(mask); } data; } draw_opaque; @@ -657,7 +657,7 @@ channel DisplayChannel : BaseChannel { Rect src_area; ropd rop_decriptor; image_scale_mode scale_mode; - QMask mask; + QMask mask @outvar(mask); } data; } draw_copy; @@ -668,28 +668,28 @@ channel DisplayChannel : BaseChannel { Rect src_area; ropd rop_decriptor; image_scale_mode scale_mode; - QMask mask; + QMask mask @outvar(mask); } @ctype(SpiceCopy) data; } draw_blend; message { DisplayBase base; struct Blackness { - QMask mask; + QMask mask @outvar(mask); } data; } draw_blackness; message { DisplayBase base; struct Whiteness { - QMask mask; + QMask mask @outvar(mask); } data; } draw_whiteness; message { DisplayBase base; struct Invers { - QMask mask; + QMask mask @outvar(mask); } data; } draw_invers; @@ -701,7 +701,7 @@ channel DisplayChannel : BaseChannel { Brush brush; uint8 rop3; image_scale_mode scale_mode; - QMask mask; + QMask mask @outvar(mask); } data; } draw_rop3; @@ -721,8 +721,8 @@ channel DisplayChannel : BaseChannel { struct Text { String *str; Rect back_area; - Brush fore_brush; - Brush back_brush; + Brush fore_brush @outvar(fore_brush); + Brush back_brush @outvar(back_brush); uint16 fore_mode; uint16 back_mode; } data; From 27238241f08736fbf3838462c32e0c457933305a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 11 Jun 2010 16:39:27 +0200 Subject: [PATCH 005/147] Add @nomarshal in a few places where we're marshalling manually --- spice.proto | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/spice.proto b/spice.proto index 32ad869..7952f9a 100644 --- a/spice.proto +++ b/spice.proto @@ -130,8 +130,8 @@ channel BaseChannel { notify_visibility visibilty; uint32 what; /* error_code/warn_code/info_code */ uint32 message_len; - uint8 message[message_len] @end; - uint8 zero @end @ctype(uint8_t) @zero; + uint8 message[message_len] @end @nomarshal; + uint8 zero @end @ctype(uint8_t) @zero @nomarshal; } notify; client: @@ -171,8 +171,8 @@ channel MainChannel : BaseChannel { pubkey_type pub_key_type @minor(2); uint32 pub_key_offset @minor(2); uint32 pub_key_size @minor(2); - uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated; - uint8 pub_key_data[pub_key_size] @minor(2) @end @ctype(uint8_t) @zero_terminated; + uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; + uint8 pub_key_data[pub_key_size] @minor(2) @end @ctype(uint8_t) @zero_terminated @nomarshal; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; @@ -455,7 +455,7 @@ struct BitmapData { struct BinaryData { uint32 data_size; - uint8 data[data_size] @end; + uint8 data[data_size] @end @nomarshal; } @ctype(SpiceQUICData); struct LZPLTData { @@ -467,7 +467,7 @@ struct LZPLTData { default: Palette *palette @nonnull @outvar(lzplt); } pal @anon; - uint8 data[data_size] @end; + uint8 data[data_size] @end @nomarshal; }; struct Surface { @@ -561,7 +561,7 @@ struct String { RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph); case RASTER_A8: RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph); - } u @end; + } u @end @nomarshal; }; channel DisplayChannel : BaseChannel { @@ -614,8 +614,8 @@ channel DisplayChannel : BaseChannel { uint32 multi_media_time; uint32 data_size; uint32 pad_size; - uint8 data[data_size] @end; - uint8 padding[pad_size] @end @ctype(uint8_t); /* Uhm, why are we sending padding over network? */ + uint8 data[data_size] @end @nomarshal; + uint8 padding[pad_size] @end @ctype(uint8_t) @nomarshal; /* Uhm, why are we sending padding over network? */ } stream_data; message { @@ -952,7 +952,7 @@ channel RecordChannel : BaseChannel { client: message { uint32 time; - uint8 data[] @end; + uint8 data[] @end @nomarshal; } @ctype(SpiceMsgcRecordPacket) data = 101; message { @@ -982,7 +982,7 @@ struct TunnelIpInfo { switch (type) { case IPv4: uint8 ipv4[4] @ctype(uint8_t); - } u @end; + } u @end @nomarshal; } @ctype(SpiceMsgTunnelIpInfo); channel TunnelChannel : BaseChannel { From 311469937848013c280614b172a93fc10493ffc4 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 17 Jun 2010 10:07:36 +0200 Subject: [PATCH 006/147] Use @ptr32 for 32bit offsets in spice.proto --- spice.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index 7952f9a..d41137f 100644 --- a/spice.proto +++ b/spice.proto @@ -1031,8 +1031,8 @@ channel TunnelChannel : BaseChannel { uint32 id; uint32 group; uint32 port; - uint32 name; - uint32 description; + uint8 *name[] @ptr32; + uint8 *description[] @ptr32; switch (type) { case IPP: TunnelIpInfo ip @ctype(SpiceMsgTunnelIpInfo); From d802d7e7f8431b9f7a79639baec73d0451f825a9 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 17 Jun 2010 11:19:31 +0200 Subject: [PATCH 007/147] Make tunnel_service_add members be of cstring array size --- spice.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index d41137f..093c808 100644 --- a/spice.proto +++ b/spice.proto @@ -1031,8 +1031,8 @@ channel TunnelChannel : BaseChannel { uint32 id; uint32 group; uint32 port; - uint8 *name[] @ptr32; - uint8 *description[] @ptr32; + uint8 *name[cstring()] @ptr32 @nocopy; + uint8 *description[cstring()] @ptr32 @nocopy; switch (type) { case IPP: TunnelIpInfo ip @ctype(SpiceMsgTunnelIpInfo); From 405900c4a204082d3326b9a92fd3d91a55f4cb34 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 18 Jun 2010 13:43:45 +0200 Subject: [PATCH 008/147] Allow multiple --include args --- spice_codegen.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spice_codegen.py b/spice_codegen.py index 0e9551f..c8d0d5f 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -108,7 +108,7 @@ parser.add_option("-k", "--keep-identical-file", action="store_true", dest="keep_identical_file", default=False, help="Print errors") parser.add_option("-i", "--include", - dest="include", default=None, metavar="FILE", + action="append", dest="includes", metavar="FILE", help="Include FILE in generated code") (options, args) = parser.parse_args() @@ -137,8 +137,9 @@ if options.assert_on_error: if options.print_error: writer.set_option("print_error") -if options.include: - writer.writeln('#include "%s"' % options.include) +if options.includes: + for i in options.includes: + writer.writeln('#include "%s"' % i) if options.generate_enums: write_enums(writer) From 41c07b938bfe53c61b28b4b9d87044766d237a10 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 18 Jun 2010 17:09:58 +0200 Subject: [PATCH 009/147] Support @as_ptr in demarshaller to avoid copying data unnecessary --- python_modules/demarshal.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index fcd6850..8b90458 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -760,7 +760,19 @@ def write_member_parser(writer, container, member, dest, scope): #TODO validate e.g. flags and enums elif t.is_array(): nelements = read_array_len(writer, member.name, t, dest, scope, handles_bytes = True) - write_array_parser(writer, nelements, t, dest, scope) + if member.has_attr("as_ptr") and t.element_type.is_fixed_nw_size(): + writer.comment("use array as pointer").newline() + writer.assign(dest.get_ref(member.name), "(%s *)in" % t.element_type.c_type()) + len_var = member.attributes["as_ptr"] + if len(len_var) > 0: + writer.assign(dest.get_ref(len_var[0]), nelements) + el_size = t.element_type.get_fixed_nw_size() + if el_size != 1: + writer.increment("in", "%s * %s" % (nelements, el_size)) + else: + writer.increment("in", "%s" % (nelements)) + else: + write_array_parser(writer, nelements, t, dest, scope) elif t.is_struct(): if member.has_end_attr(): dest2 = dest.child_at_end(writer, t) From 98d3b1d5c31720b5d47e1084c7c93503fb5291fd Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 18 Jun 2010 17:11:39 +0200 Subject: [PATCH 010/147] Make cursor data @as_ptr to avoid copying data --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index 093c808..6750d2d 100644 --- a/spice.proto +++ b/spice.proto @@ -867,7 +867,7 @@ struct CursorHeader { struct Cursor { cursor_flags flags; CursorHeader header; - uint8 data[] @end; + uint8 data[] @end @as_ptr(data_size); }; channel CursorChannel : BaseChannel { From 4b670111a0109ef0e026ac810ef2bce8cc8c6149 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 18 Jun 2010 17:12:07 +0200 Subject: [PATCH 011/147] Make ping data @as_ptr to avoid copying data --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index 6750d2d..bad9ed3 100644 --- a/spice.proto +++ b/spice.proto @@ -111,7 +111,7 @@ channel BaseChannel { message { uint32 id; uint64 timestamp; - uint8 data[] @end @ctype(uint8_t); + uint8 data[] @end @ctype(uint8_t) @as_ptr(data_len); } ping; message { From a0a19077161242be4eec4f78583b009d0b478d31 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 18 Jun 2010 17:12:31 +0200 Subject: [PATCH 012/147] Make sound data @as_ptr to avoid copying data --- spice.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spice.proto b/spice.proto index bad9ed3..6cedfac 100644 --- a/spice.proto +++ b/spice.proto @@ -921,13 +921,13 @@ channel PlaybackChannel : BaseChannel { server: message { uint32 time; - uint8 data[] @end; + uint8 data[] @end @as_ptr(data_size); } @ctype(SpiceMsgPlaybackPacket) data = 101; message { uint32 time; audio_data_mode mode; - uint8 data[] @end; + uint8 data[] @end @as_ptr(data_size); } mode; message { @@ -952,13 +952,13 @@ channel RecordChannel : BaseChannel { client: message { uint32 time; - uint8 data[] @end @nomarshal; + uint8 data[] @end @nomarshal @as_ptr(data_size); } @ctype(SpiceMsgcRecordPacket) data = 101; message { uint32 time; audio_data_mode mode; - uint8 data[] @end; + uint8 data[] @end @as_ptr(data_size); } mode; message { From b49b77d8e9bd535d88e1ad72d6bdbbddba015040 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 18 Jun 2010 21:10:25 +0200 Subject: [PATCH 013/147] Make generated marshallers build on win32 --- python_modules/marshal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 23b029a..4cbf942 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -18,6 +18,7 @@ def write_includes(writer): writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") + writer.writeln("#pragma warning(disable:4018)") writer.writeln("#endif") writer.newline() From e8ee21132bda530122a9bb5e33d117f5e7937204 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Sun, 20 Jun 2010 15:24:49 +0300 Subject: [PATCH 014/147] applying zlib compression over glz on WAN connection --- spice.proto | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spice.proto b/spice.proto index 6cedfac..84d3fa9 100644 --- a/spice.proto +++ b/spice.proto @@ -293,6 +293,7 @@ enum8 image_type { SURFACE, JPEG, FROM_CACHE_LOSSLESS, + ZLIB_GLZ_RGB, }; flags8 image_flags { @@ -470,6 +471,12 @@ struct LZPLTData { uint8 data[data_size] @end @nomarshal; }; +struct ZlibGlzRGBData { + uint32 glz_data_size; + uint32 data_size; + uint8 data[data_size] @end @nomarshal; +} @ctype(SpiceZlibGlzRGBData); + struct Surface { uint32 surface_id; }; @@ -491,6 +498,8 @@ struct Image { BinaryData binary_data @ctype(SpiceQUICData); case LZ_PLT: LZPLTData lzplt_data @ctype(SpiceLZPLTData); + case ZLIB_GLZ_RGB: + ZlibGlzRGBData zlib_glz_data @ctype(SpiceZlibGlzRGBData); case SURFACE: Surface surface_data; } u @end; From 1f3c193cebafd44cd588aca14ccbf2c309115f65 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Sun, 20 Jun 2010 17:18:56 +0300 Subject: [PATCH 015/147] Lossy compression of RGBA images (on WAN connection) The RGB channels are compressed using JPEG. The alpha channel is compressed using LZ. --- spice.proto | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/spice.proto b/spice.proto index 84d3fa9..dbd3dde 100644 --- a/spice.proto +++ b/spice.proto @@ -294,6 +294,7 @@ enum8 image_type { JPEG, FROM_CACHE_LOSSLESS, ZLIB_GLZ_RGB, + JPEG_ALPHA, }; flags8 image_flags { @@ -321,6 +322,10 @@ flags8 bitmap_flags { TOP_DOWN, }; +flags8 jpeg_alpha_flags { + TOP_DOWN, +}; + enum8 image_scale_mode { INTERPOLATE, NEAREST, @@ -477,6 +482,13 @@ struct ZlibGlzRGBData { uint8 data[data_size] @end @nomarshal; } @ctype(SpiceZlibGlzRGBData); +struct JPEGAlphaData { + jpeg_alpha_flags flags; + uint32 jpeg_size; + uint32 data_size; + uint8 data[data_size] @end @nomarshal; +} @ctype(SpiceJPEGAlphaData); + struct Surface { uint32 surface_id; }; @@ -500,6 +512,8 @@ struct Image { LZPLTData lzplt_data @ctype(SpiceLZPLTData); case ZLIB_GLZ_RGB: ZlibGlzRGBData zlib_glz_data @ctype(SpiceZlibGlzRGBData); + case JPEG_ALPHA: + JPEGAlphaData jpeg_alpha_data @ctype(SpiceJPEGAlphaData); case SURFACE: Surface surface_data; } u @end; From f9610c1a658d78660cfca01279350d2c235b8dba Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 22 Jun 2010 10:53:24 +0200 Subject: [PATCH 016/147] Add destructor for demarshalled messages This is required because we don't want to free messages that just refer to the unparsed message (like SpiceMsgData). Also, in the future we might need it for more complex demarshalling. --- python_modules/demarshal.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 8b90458..b1c4135 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -41,9 +41,10 @@ def write_parser_helpers(writer): writer.newline() writer.statement("typedef struct PointerInfo PointerInfo") + writer.statement("typedef void (*message_destructor_t)(uint8_t *message)"); writer.statement("typedef uint8_t * (*parse_func_t)(uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *ptr_info, int minor)") - writer.statement("typedef uint8_t * (*parse_msg_func_t)(uint8_t *message_start, uint8_t *message_end, int minor, size_t *size_out)") - writer.statement("typedef uint8_t * (*spice_parse_channel_func_t)(uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out)") + writer.statement("typedef uint8_t * (*parse_msg_func_t)(uint8_t *message_start, uint8_t *message_end, int minor, size_t *size_out, message_destructor_t *free_message)") + writer.statement("typedef uint8_t * (*spice_parse_channel_func_t)(uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message)") writer.newline() writer.begin_block("struct PointerInfo") @@ -820,6 +821,13 @@ def write_ptr_info_check(writer): writer.error_check("end == NULL") writer.newline() +def write_nofree(writer): + if writer.is_generated("helper", "nofree"): + return + writer = writer.function_helper() + scope = writer.function("nofree", "static void", "uint8_t *data") + writer.end_block() + def write_msg_parser(writer, message): msg_name = message.c_name() function_name = "parse_%s" % msg_name @@ -833,7 +841,7 @@ def write_msg_parser(writer, message): writer.newline() parent_scope = writer.function(function_name, "uint8_t *", - "uint8_t *message_start, uint8_t *message_end, int minor, size_t *size", True) + "uint8_t *message_start, uint8_t *message_end, int minor, size_t *size, message_destructor_t *free_message", True) parent_scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); parent_scope.variable_def("uint8_t *", "start = message_start"); parent_scope.variable_def("uint8_t *", "data = NULL"); @@ -860,8 +868,10 @@ def write_msg_parser(writer, message): writer.newline().comment("Validated extents and calculated size").newline() if message.has_attr("nocopy"): + write_nofree(writer) writer.assign("data", "message_start") writer.assign("*size", "message_end - message_start") + writer.assign("*free_message", "nofree") else: writer.assign("data", "(uint8_t *)malloc(mem_size)") writer.error_check("data == NULL") @@ -882,6 +892,7 @@ def write_msg_parser(writer, message): writer.newline() writer.assign("*size", "end - data") + writer.assign("*free_message", "(message_destructor_t) free") writer.statement("return data") writer.newline() @@ -922,7 +933,7 @@ def write_channel_parser(writer, channel, server): writer.newline() scope = writer.function(function_name, "uint8_t *", - "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out") + "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") helpers = writer.function_helper() @@ -944,7 +955,7 @@ def write_channel_parser(writer, channel, server): for r in ranges: d = d + 1 with writer.if_block("message_type >= %d && message_type < %d" % (r[0], r[1]), d > 1, False): - writer.statement("return funcs%d[message_type-%d](message_start, message_end, minor, size_out)" % (d, r[0])) + writer.statement("return funcs%d[message_type-%d](message_start, message_end, minor, size_out, free_message)" % (d, r[0])) writer.newline() writer.statement("return NULL") @@ -1006,7 +1017,7 @@ def write_full_protocol_parser(writer, is_server): function_name = "spice_parse_reply" scope = writer.function(function_name, "uint8_t *", - "uint8_t *message_start, uint8_t *message_end, uint32_t channel, uint16_t message_type, int minor, size_t *size_out") + "uint8_t *message_start, uint8_t *message_end, uint32_t channel, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") scope.variable_def("spice_parse_channel_func_t", "func" ) if is_server: @@ -1015,7 +1026,7 @@ def write_full_protocol_parser(writer, is_server): writer.assign("func", "spice_get_client_channel_parser(channel, NULL)") with writer.if_block("func != NULL"): - writer.statement("return func(message_start, message_end, message_type, minor, size_out)") + writer.statement("return func(message_start, message_end, message_type, minor, size_out, free_message)") writer.statement("return NULL") writer.end_block() From 38412df9aaac82865f7251ee67623fe48d034940 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 22 Jun 2010 10:32:08 +0200 Subject: [PATCH 017/147] Support creating marshallers that are called indirectly This is needed if we want to switch marshallers depending on what major version the remote side has. --- python_modules/marshal.py | 31 +++++++++++++++++++++++++------ spice_codegen.py | 7 +++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 4cbf942..4279cf0 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -310,7 +310,7 @@ def write_container_marshaller(writer, container, src): writer.out_prefix = saved_out_prefix write_member_marshaller(writer, container, m, src, scope) -def write_message_marshaller(writer, message, is_server): +def write_message_marshaller(writer, message, is_server, private): writer.out_prefix = "" function_name = "spice_marshall_" + message.c_name() if writer.is_generated("marshaller", function_name): @@ -323,10 +323,11 @@ def write_message_marshaller(writer, message, is_server): n = map(lambda name: ", SpiceMarshaller **%s" % name, names) names_args = "".join(n) - writer.header.writeln("void " + function_name + "(SpiceMarshaller *m, %s *msg" % message.c_type() + names_args + ");") + if not private: + writer.header.writeln("void " + function_name + "(SpiceMarshaller *m, %s *msg" % message.c_type() + names_args + ");") scope = writer.function(function_name, - "void", + "static void" if private else "void", "SpiceMarshaller *m, %s *msg" % message.c_type() + names_args) scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") @@ -341,18 +342,36 @@ def write_message_marshaller(writer, message, is_server): writer.end_block() writer.newline() + return function_name -def write_protocol_marshaller(writer, proto, is_server): +def write_protocol_marshaller(writer, proto, is_server, private_marshallers): + functions = {} for c in proto.channels: channel = c.channel_type if is_server: for m in channel.client_messages: message = m.message_type - write_message_marshaller(writer, message, is_server) + f = write_message_marshaller(writer, message, is_server, private_marshallers) + functions[f] = True else: for m in channel.server_messages: message = m.message_type - write_message_marshaller(writer, message, is_server) + f= write_message_marshaller(writer, message, is_server, private_marshallers) + functions[f] = True + + if private_marshallers: + scope = writer.function("spice_message_marshallers_get", + "SpiceMessageMarshallers *", + "void") + writer.writeln("static SpiceMessageMarshallers marshallers = {NULL};").newline() + for f in sorted(functions.keys()): + member = f[len("spice_marshall_"):] + writer.assign("marshallers.%s" % member, f) + + writer.newline() + writer.statement("return &marshallers") + writer.end_block() + writer.newline() def write_trailer(writer): writer.header.writeln("#endif") diff --git a/spice_codegen.py b/spice_codegen.py index c8d0d5f..0034250 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -86,6 +86,9 @@ parser.add_option("-d", "--generate-demarshallers", parser.add_option("-m", "--generate-marshallers", action="store_true", dest="generate_marshallers", default=False, help="Generate message marshallers") +parser.add_option("-P", "--private-marshallers", + action="store_true", dest="private_marshallers", default=False, + help="Generate private message marshallers") parser.add_option("-M", "--generate-struct-marshaller", action="append", dest="struct_marshallers", help="Generate struct marshallers") @@ -163,9 +166,9 @@ if options.generate_marshallers: print >> sys.stderr, "Must specify client and/or server" sys.exit(1) if options.server: - marshal.write_protocol_marshaller(writer, proto, False) + marshal.write_protocol_marshaller(writer, proto, False, options.private_marshallers) if options.client: - marshal.write_protocol_marshaller(writer, proto, True) + marshal.write_protocol_marshaller(writer, proto, True, options.private_marshallers) if options.struct_marshallers: for structname in options.struct_marshallers: From 2523cec8c4890f977ee5569d9280d54108790674 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 22 Jun 2010 16:01:57 +0200 Subject: [PATCH 018/147] Support extra prefix in code generators This is require when we add a new spice.proto for the old (major 1) protocol description. --- python_modules/codegen.py | 1 + python_modules/demarshal.py | 10 +++++----- python_modules/marshal.py | 2 +- spice_codegen.py | 4 ++++ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 5bb659a..af6636b 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -113,6 +113,7 @@ class CodeWriter: writer.at_line_start = self.at_line_start writer.generated = self.generated writer.options = self.options + writer.public_prefix = self.public_prefix return writer; diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index b1c4135..8d86e84 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -966,9 +966,9 @@ def write_channel_parser(writer, channel, server): def write_get_channel_parser(writer, channel_parsers, max_channel, is_server): writer.newline() if is_server: - function_name = "spice_get_server_channel_parser" + function_name = "spice_get_server_channel_parser" + writer.public_prefix else: - function_name = "spice_get_client_channel_parser" + function_name = "spice_get_client_channel_parser" + writer.public_prefix scope = writer.function(function_name, "spice_parse_channel_func_t", @@ -1015,15 +1015,15 @@ def write_full_protocol_parser(writer, is_server): function_name = "spice_parse_msg" else: function_name = "spice_parse_reply" - scope = writer.function(function_name, + scope = writer.function(function_name + writer.public_prefix, "uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint32_t channel, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") scope.variable_def("spice_parse_channel_func_t", "func" ) if is_server: - writer.assign("func", "spice_get_server_channel_parser(channel, NULL)") + writer.assign("func", "spice_get_server_channel_parser%s(channel, NULL)" % writer.public_prefix) else: - writer.assign("func", "spice_get_client_channel_parser(channel, NULL)") + writer.assign("func", "spice_get_client_channel_parser%s(channel, NULL)" % writer.public_prefix) with writer.if_block("func != NULL"): writer.statement("return func(message_start, message_end, message_type, minor, size_out, free_message)") diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 4279cf0..76081a9 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -360,7 +360,7 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): functions[f] = True if private_marshallers: - scope = writer.function("spice_message_marshallers_get", + scope = writer.function("spice_message_marshallers_get" + writer.public_prefix, "SpiceMessageMarshallers *", "void") writer.writeln("static SpiceMessageMarshallers marshallers = {NULL};").newline() diff --git a/spice_codegen.py b/spice_codegen.py index 0034250..1d2314a 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -113,6 +113,8 @@ parser.add_option("-k", "--keep-identical-file", parser.add_option("-i", "--include", action="append", dest="includes", metavar="FILE", help="Include FILE in generated code") +parser.add_option("--prefix", dest="prefix", + help="set public symbol prefix", default="") (options, args) = parser.parse_args() @@ -134,6 +136,8 @@ writer = codegen.CodeWriter() writer.header = codegen.CodeWriter() writer.set_option("source", os.path.basename(proto_file)) +writer.public_prefix = options.prefix + if options.assert_on_error: writer.set_option("assert_on_error") From 9f3a36f3f939c14da4e2fa1f98f3ebfd50dcf9c7 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 22 Jun 2010 16:03:02 +0200 Subject: [PATCH 019/147] Make internal generated marshaller functions static --- python_modules/demarshal.py | 6 +++--- python_modules/marshal.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 8d86e84..eee659f 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -149,7 +149,7 @@ def write_validate_struct_function(writer, struct): writer.set_is_generated("validator", validate_function) writer = writer.function_helper() - scope = writer.function(validate_function, "intptr_t", "uint8_t *message_start, uint8_t *message_end, SPICE_ADDRESS offset, int minor") + scope = writer.function(validate_function, "static intptr_t", "uint8_t *message_start, uint8_t *message_end, SPICE_ADDRESS offset, int minor") scope.variable_def("uint8_t *", "start = message_start + offset") scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); scope.variable_def("size_t", "mem_size", "nw_size"); @@ -661,7 +661,7 @@ def write_parse_ptr_function(writer, target_type): writer.set_is_generated("parser", parse_function) writer = writer.function_helper() - scope = writer.function(parse_function, "uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *this_ptr_info, int minor") + scope = writer.function(parse_function, "static uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *this_ptr_info, int minor") scope.variable_def("uint8_t *", "in = message_start + this_ptr_info->offset") scope.variable_def("uint8_t *", "end") @@ -932,7 +932,7 @@ def write_channel_parser(writer, channel, server): function_name = "parse_%s_msgc" % channel.name writer.newline() scope = writer.function(function_name, - "uint8_t *", + "static uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") helpers = writer.function_helper() diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 76081a9..c4bb896 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -115,7 +115,7 @@ def write_marshal_ptr_function(writer, target_type): writer.header = header writer.out_prefix = "" if target_type.is_array(): - scope = writer.function(marshal_function, "void *", "SpiceMarshaller *m, %s_t *ptr, int count" % target_type.element_type.primitive_type() + names_args) + scope = writer.function(marshal_function, "SPICE_GNUC_UNUSED static void *", "SpiceMarshaller *m, %s_t *ptr, int count" % target_type.element_type.primitive_type() + names_args) else: scope = writer.function(marshal_function, "void *", "SpiceMarshaller *m, %s *ptr" % target_type.c_type() + names_args) header.writeln("void *" + marshal_function + "(SpiceMarshaller *m, %s *msg" % target_type.c_type() + names_args + ");") From ee944c8314287a1037b9ede077583b1cec31ea07 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 22 Jun 2010 16:03:34 +0200 Subject: [PATCH 020/147] Add support for @virtual markup in spice protocol This means the member is not sent on the network at all. Instead its initialized to the attribute argument when demarshalled. This is useful for backwards compatibility support. --- python_modules/demarshal.py | 7 +++++++ python_modules/marshal.py | 3 +++ python_modules/ptypes.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index eee659f..a0697a9 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -400,6 +400,9 @@ def write_validate_item(writer, container, item, scope, parent_scope, start, def write_validate_member(writer, container, member, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): + if member.has_attr("virtual"): + return + if member.has_minor_attr(): prefix = "if (minor >= %s)" % (member.get_minor_attr()) newline = False @@ -740,6 +743,10 @@ def write_parse_pointer(writer, t, at_end, dest, member_name, is_64bit, scope): writer.statement("n_ptr++") def write_member_parser(writer, container, member, dest, scope): + if member.has_attr("virtual"): + writer.assign(dest.get_ref(member.name), member.attributes["virtual"][0]) + return + if member.is_switch(): write_switch_parser(writer, container, member, dest, scope) return diff --git a/python_modules/marshal.py b/python_modules/marshal.py index c4bb896..c5afd7c 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -266,6 +266,9 @@ def write_switch_marshaller(writer, container, switch, src, scope): def write_member_marshaller(writer, container, member, src, scope): if member.has_attr("outvar"): writer.out_prefix = "%s_%s" % (member.attributes["outvar"][0], writer.out_prefix) + if member.has_attr("virtual"): + writer.comment("Don't marshall @virtual %s" % member.name).newline() + return if member.has_attr("nomarshal"): writer.comment("Don't marshall @nomarshal %s" % member.name).newline() return diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index fe8a321..101538c 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -518,9 +518,13 @@ class Member(Containee): return self.has_end_attr() def is_fixed_nw_size(self): + if self.has_attr("virtual"): + return True return self.member_type.is_fixed_nw_size() def get_fixed_nw_size(self): + if self.has_attr("virtual"): + return 0 size = self.member_type.get_fixed_nw_size() if self.has_minor_attr(): minor = self.get_minor_attr() From fbf883085f0594fb1c029267e7ec70dad536b4a3 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 22 Jun 2010 16:31:40 +0200 Subject: [PATCH 021/147] Add spice1.proto describing the 0.4 version of the network protocolx --- spice1.proto | 934 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 934 insertions(+) create mode 100644 spice1.proto diff --git a/spice1.proto b/spice1.proto new file mode 100644 index 0000000..0efe74b --- /dev/null +++ b/spice1.proto @@ -0,0 +1,934 @@ +/* built in types: + int8, uint8, 16, 32, 64 +*/ + +typedef fixed28_4 int32 @ctype(SPICE_FIXED28_4); + +struct Point { + int32 x; + int32 y; +}; + +struct Point16 { + int16 x; + int16 y; +}; + +struct PointFix { + fixed28_4 x; + fixed28_4 y; +}; + +struct Rect { + int32 top; + int32 left; + int32 bottom; + int32 right; +}; + +enum32 link_err { + OK, + ERROR, + INVALID_MAGIC, + INVALID_DATA, + VERSION_MISMATCH, + NEED_SECURED, + NEED_UNSECURED, + PERMISSION_DENIED, + BAD_CONNECTION_ID, + CHANNEL_NOT_AVAILABLE +}; + +enum32 warn_code { + WARN_GENERAL +} @prefix(SPICE_); + +enum32 info_code { + INFO_GENERAL +} @prefix(SPICE_); + +flags32 migrate_flags { + NEED_FLUSH, + NEED_DATA_TRANSFER +} @prefix(SPICE_MIGRATE_); + +enum32 notify_severity { + INFO, + WARN, + ERROR, +}; + +enum32 notify_visibility { + LOW, + MEDIUM, + HIGH, +}; + +flags32 mouse_mode { + SERVER, + CLIENT, +}; + +enum16 pubkey_type { + INVALID, + RSA, + RSA2, + DSA, + DSA1, + DSA2, + DSA3, + DSA4, + DH, + EC, +}; + +message Empty { +}; + +message Data { + uint8 data[] @end @ctype(uint8_t); +} @nocopy; + +struct ChannelWait { + uint8 channel_type; + uint8 channel_id; + uint64 message_serial; +} @ctype(SpiceWaitForChannel); + +channel BaseChannel { + server: + message { + migrate_flags flags; + } migrate; + + Data migrate_data; + + message { + uint32 generation; + uint32 window; + } set_ack; + + message { + uint32 id; + uint64 timestamp; + uint8 data[] @end @ctype(uint8_t) @as_ptr(data_len); + } ping; + + message { + uint8 wait_count; + ChannelWait wait_list[wait_count] @end; + } wait_for_channels; + + message { + uint64 time_stamp; + link_err reason; + } @ctype(SpiceMsgDisconnect) disconnecting; + + message { + uint64 time_stamp; + notify_severity severity; + notify_visibility visibilty; + uint32 what; /* error_code/warn_code/info_code */ + uint32 message_len; + uint8 message[message_len] @end @nomarshal; + uint8 zero @end @ctype(uint8_t) @zero @nomarshal; + } notify; + + client: + message { + uint32 generation; + } ack_sync; + + Empty ack; + + message { + uint32 id; + uint64 timestamp; + } @ctype(SpiceMsgPing) pong; + + Empty migrate_flush_mark; + + Data migrate_data; + + message { + uint64 time_stamp; + link_err reason; + } @ctype(SpiceMsgDisconnect) disconnecting; +}; + +struct ChannelId { + uint8 type; + uint8 id; +}; + +channel MainChannel : BaseChannel { + server: + message { + uint16 port; + uint16 sport; + uint32 host_offset; + uint32 host_size; + pubkey_type pub_key_type @minor(2); + uint32 pub_key_offset @minor(2); + uint32 pub_key_size @minor(2); + uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; + uint8 pub_key_data[pub_key_size] @minor(2) @end @ctype(uint8_t) @zero_terminated @nomarshal; + } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; + + Empty migrate_cancel; + + message { + uint32 session_id; + uint32 display_channels_hint; + uint32 supported_mouse_modes; + uint32 current_mouse_mode; + uint32 agent_connected; + uint32 agent_tokens; + uint32 multi_media_time; + uint32 ram_hint; + } init; + + message { + uint32 num_of_channels; + ChannelId channels[num_of_channels] @end; + } @ctype(SpiceMsgChannels) channels_list; + + message { + mouse_mode supported_modes; + mouse_mode current_mode @unique_flag; + } mouse_mode; + + message { + uint32 time; + } @ctype(SpiceMsgMainMultiMediaTime) multi_media_time; + + Empty agent_connected; + + message { + link_err error_code; + } @ctype(SpiceMsgMainAgentDisconnect) agent_disconnected; + + Data agent_data; + + message { + uint32 num_tokens; + } @ctype(SpiceMsgMainAgentTokens) agent_token; + + message { + uint16 port; + uint16 sport; + uint32 host_offset; + uint32 host_size; + uint32 cert_subject_offset; + uint32 cert_subject_size; + uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated; + uint8 cert_subject_data[cert_subject_size] @end @ctype(uint8_t) @zero_terminated; + } @ctype(SpiceMsgMainMigrationSwitchHost) migrate_switch_host; + + client: + message { + uint64 cache_size; + } @ctype(SpiceMsgcClientInfo) client_info = 101; + + Empty migrate_connected; + + Empty migrate_connect_error; + + Empty attach_channels; + + message { + mouse_mode mode; + } mouse_mode_request; + + message { + uint32 num_tokens; + } agent_start; + + Data agent_data; + + message { + uint32 num_tokens; + } @ctype(SpiceMsgcMainAgentTokens) agent_token; +}; + +enum32 clip_type { + NONE, + RECTS, + PATH, +}; + +flags32 path_flags { /* TODO: C enum names changes */ + BEGIN = 0, + END = 1, + CLOSE = 3, + BEZIER = 4, +} @prefix(SPICE_PATH_); + +enum32 video_codec_type { + MJPEG = 1, +}; + +flags32 stream_flags { + TOP_DOWN = 0, +}; + +enum32 brush_type { + NONE, + SOLID, + PATTERN, +}; + +flags8 mask_flags { + INVERS, +}; + +enum8 image_type { + BITMAP, + QUIC, + RESERVED, + LZ_PLT = 100, + LZ_RGB, + GLZ_RGB, + FROM_CACHE, +}; + +flags8 image_flags { + CACHE_ME, +}; + +enum8 bitmap_fmt { + INVALID, + 1BIT_LE, + 1BIT_BE, + 4BIT_LE, + 4BIT_BE, + 8BIT /* 8bit indexed mode */, + 16BIT, /* 0555 mode */ + 24BIT /* 3 byte, brg */, + 32BIT /* 4 byte, xrgb in little endian format */, + RGBA /* 4 byte, argb in little endian format */ +}; + +flags8 bitmap_flags { + PAL_CACHE_ME, + PAL_FROM_CACHE, + TOP_DOWN, +}; + +enum8 image_scale_mode { + INTERPOLATE, + NEAREST, +}; + +flags16 ropd { + INVERS_SRC, + INVERS_BRUSH, + INVERS_DEST, + OP_PUT, + OP_OR, + OP_AND, + OP_XOR, + OP_BLACKNESS, + OP_WHITENESS, + OP_INVERS, + INVERS_RES, +}; + +flags8 line_flags { + STYLED = 3, + START_WITH_GAP = 2, +}; + +enum8 line_cap { + ROUND, + SQUARE, + BUTT, +}; + +enum8 line_join { + ROUND, + BEVEL, + MITER, +}; + +flags16 string_flags { + RASTER_A1, + RASTER_A4, + RASTER_A8, + RASTER_TOP_DOWN, +}; + +enum8 resource_type { + INVALID, + PIXMAP +} @prefix(SPICE_RES_TYPE_); + +struct ClipRects { + uint32 num_rects; + Rect rects[num_rects] @end; +}; + +struct PathSegment { + path_flags flags; + uint32 count; + PointFix points[count] @end; +} @ctype(SpicePathSeg); + +struct Path { + uint32 size; + PathSegment segments[bytes(size)] @end; +}; + +struct Clip { + clip_type type; + switch (type) { + case NONE: + uint64 data @zero; + case RECTS: + ClipRects *data @outvar(cliprects); + case PATH: + Path *data @outvar(clippath); + } u @anon; +}; + +struct DisplayBase { + uint32 surface_id @virtual(0); + Rect box; + Clip clip; +} @ctype(SpiceMsgDisplayBase); + +struct ResourceID { + uint8 type; + uint64 id; +}; + +struct WaitForChannel { + uint8 channel_type; + uint8 channel_id; + uint64 message_serial; +}; + +struct Palette { + uint64 unique; + uint16 num_ents; + uint32 ents[num_ents] @end; +}; + +struct BitmapData { + bitmap_fmt format; + bitmap_flags flags; + uint32 x; + uint32 y; + uint32 stride; + switch (flags) { + case PAL_FROM_CACHE: + uint64 palette; + default: + Palette *palette @outvar(bitmap); + } pal @anon; + uint8 *data[image_size(8, stride, y)] @nocopy; /* pointer to array, not array of pointers as in C */ +} @ctype(SpiceBitmap); + +struct BinaryData { + uint32 data_size; + uint8 data[data_size] @end @nomarshal; +} @ctype(SpiceQUICData); + +struct LZPLTData { + bitmap_flags flags; + uint32 data_size; + switch (flags) { + case PAL_FROM_CACHE: + uint64 palette; + default: + Palette *palette @nonnull @outvar(lzplt); + } pal @anon; + uint8 data[data_size] @end @nomarshal; +}; + +struct Image { + uint64 id; + image_type type; + image_flags flags; + uint32 width; + uint32 height; + + switch (type) { + case BITMAP: + BitmapData bitmap_data @ctype(SpiceBitmap); + case QUIC: + case LZ_RGB: + case GLZ_RGB: + BinaryData binary_data @ctype(SpiceQUICData); + case LZ_PLT: + LZPLTData lzplt_data @ctype(SpiceLZPLTData); + } u @end; +} @ctype(SpiceImageDescriptor); + +struct Pattern { + Image *pat @nonnull; + Point pos; +}; + +struct Brush { + brush_type type; + switch (type) { + case SOLID: + uint32 color; + case PATTERN: + Pattern pattern; + } u @fixedsize; +}; + +struct QMask { + mask_flags flags; + Point pos; + Image *bitmap; +}; + +struct LineAttr { + line_flags flags; + line_join join_style; + line_cap end_style; + uint8 style_nseg; + fixed28_4 width; + fixed28_4 miter_limit; + fixed28_4 *style[style_nseg]; +}; + +struct RasterGlyphA1 { + Point render_pos; + Point glyph_origin; + uint16 width; + uint16 height; + uint8 data[image_size(1, width, height)] @end; +} @ctype(SpiceRasterGlyph); + +struct RasterGlyphA4 { + Point render_pos; + Point glyph_origin; + uint16 width; + uint16 height; + uint8 data[image_size(4, width, height)] @end; +} @ctype(SpiceRasterGlyph); + +struct RasterGlyphA8 { + Point render_pos; + Point glyph_origin; + uint16 width; + uint16 height; + uint8 data[image_size(8, width, height)] @end; +} @ctype(SpiceRasterGlyph); + +struct String { + uint16 length; + string_flags flags; /* Special: Only one of a1/a4/a8 set */ + switch (flags) { + case RASTER_A1: + RasterGlyphA1 glyphs[length] @ctype(SpiceRasterGlyph); + case RASTER_A4: + RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph); + case RASTER_A8: + RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph); + } u @end @nomarshal; +}; + +channel DisplayChannel : BaseChannel { + server: + message { + uint32 x_res; + uint32 y_res; + uint32 bits; + } mode = 101; + + Empty mark; + Empty reset; + + message { + DisplayBase base; + Point src_pos; + } copy_bits; + + message { + uint16 count; + ResourceID resources[count] @end; + } @ctype(SpiceResourceList) inval_list; + + message { + uint8 wait_count; + WaitForChannel wait_list[wait_count] @end; + } @ctype(SpiceMsgWaitForChannels) inval_all_pixmaps; + + message { + uint64 id; + } @ctype(SpiceMsgDisplayInvalOne) inval_palette; + + Empty inval_all_palettes; + + message { + uint32 surface_id @virtual(0); + uint32 id; + stream_flags flags; + video_codec_type codec_type; + uint64 stamp; + uint32 stream_width; + uint32 stream_height; + uint32 src_width; + uint32 src_height; + Rect dest; + Clip clip; + } stream_create = 122; + + message { + uint32 id; + uint32 multi_media_time; + uint32 data_size; + uint32 pad_size; + uint8 data[data_size] @end @nomarshal; + uint8 padding[pad_size] @end @ctype(uint8_t) @nomarshal; /* Uhm, why are we sending padding over network? */ + } stream_data; + + message { + uint32 id; + Clip clip; + } stream_clip; + + message { + uint32 id; + } stream_destroy; + + Empty stream_destroy_all; + + message { + DisplayBase base; + struct Fill { + Brush brush @outvar(brush); + uint16 rop_decriptor; + QMask mask @outvar(mask); + } data; + } draw_fill = 302; + + message { + DisplayBase base; + struct Opaque { + Image *src_bitmap; + Rect src_area; + Brush brush; + ropd rop_decriptor; + image_scale_mode scale_mode; + QMask mask @outvar(mask); + } data; + } draw_opaque; + + message { + DisplayBase base; + struct Copy { + Image *src_bitmap; + Rect src_area; + ropd rop_decriptor; + image_scale_mode scale_mode; + QMask mask @outvar(mask); + } data; + } draw_copy; + + message { + DisplayBase base; + struct Blend { + Image *src_bitmap; + Rect src_area; + ropd rop_decriptor; + image_scale_mode scale_mode; + QMask mask @outvar(mask); + } @ctype(SpiceCopy) data; + } draw_blend; + + message { + DisplayBase base; + struct Blackness { + QMask mask @outvar(mask); + } data; + } draw_blackness; + + message { + DisplayBase base; + struct Whiteness { + QMask mask @outvar(mask); + } data; + } draw_whiteness; + + message { + DisplayBase base; + struct Invers { + QMask mask @outvar(mask); + } data; + } draw_invers; + + message { + DisplayBase base; + struct Rop3 { + Image *src_bitmap; + Rect src_area; + Brush brush; + uint8 rop3; + image_scale_mode scale_mode; + QMask mask @outvar(mask); + } data; + } draw_rop3; + + message { + DisplayBase base; + struct Stroke { + Path *path; + LineAttr attr; + Brush brush; + uint16 fore_mode; + uint16 back_mode; + } data; + } draw_stroke; + + message { + DisplayBase base; + struct Text { + String *str; + Rect back_area; + Brush fore_brush @outvar(fore_brush); + Brush back_brush @outvar(back_brush); + uint16 fore_mode; + uint16 back_mode; + } data; + } draw_text; + + message { + DisplayBase base; + struct Transparent { + Image *src_bitmap; + Rect src_area; + uint32 src_color; + uint32 true_color; + } data; + } draw_transparent; + + message { + DisplayBase base; + struct AlphaBlnd { + int8 alpha_flags @virtual(0); + uint8 alpha; + Image *src_bitmap; + Rect src_area; + } data; + } draw_alpha_blend; + + client: + message { + uint8 pixmap_cache_id; + int64 pixmap_cache_size; //in pixels + uint8 glz_dictionary_id; + int32 glz_dictionary_window_size; // in pixels + } init = 101; +}; + +flags32 keyboard_modifier_flags { + SCROLL_LOCK, + NUM_LOCK, + CAPS_LOCK +}; + +enum32 mouse_button { + INVALID, + LEFT, + MIDDLE, + RIGHT, + UP, + DOWN, +}; + +flags32 mouse_button_mask { + LEFT, + MIDDLE, + RIGHT +}; + +channel InputsChannel : BaseChannel { + client: + message { + uint32 code; + } @ctype(SpiceMsgcKeyDown) key_down = 101; + + message { + uint32 code; + } @ctype(SpiceMsgcKeyUp) key_up; + + message { + keyboard_modifier_flags modifiers; + } @ctype(SpiceMsgcKeyModifiers) key_modifiers; + + message { + int32 dx; + int32 dy; + mouse_button_mask buttons_state; + } @ctype(SpiceMsgcMouseMotion) mouse_motion = 111; + + message { + uint32 x; + uint32 y; + mouse_button_mask buttons_state; + uint8 display_id; + } @ctype(SpiceMsgcMousePosition) mouse_position; + + message { + mouse_button button; + mouse_button_mask buttons_state; + } @ctype(SpiceMsgcMousePress) mouse_press; + + message { + mouse_button button; + mouse_button_mask buttons_state; + } @ctype(SpiceMsgcMouseRelease) mouse_release; + + server: + message { + keyboard_modifier_flags keyboard_modifiers; + } init = 101; + + message { + keyboard_modifier_flags modifiers; + } key_modifiers; + + Empty mouse_motion_ack = 111; +}; + +enum16 cursor_type { + ALPHA, + MONO, + COLOR4, + COLOR8, + COLOR16, + COLOR24, + COLOR32, +}; + +flags32 cursor_flags { + NONE, /* Means no cursor */ + CACHE_ME, + FROM_CACHE, +}; + +struct CursorHeader { + uint64 unique; + cursor_type type; + uint16 width; + uint16 height; + uint16 hot_spot_x; + uint16 hot_spot_y; +}; + +struct Cursor { + cursor_flags flags; + CursorHeader header; + uint8 data[] @end @as_ptr(data_size); +}; + +channel CursorChannel : BaseChannel { + server: + message { + Point16 position; + uint16 trail_length; + uint16 trail_frequency; + uint8 visible; + Cursor cursor; + } init = 101; + + Empty reset; + + message { + Point16 position; + uint8 visible; + Cursor cursor; + } set; + + message { + Point16 position; + } move; + + Empty hide; + + message { + uint16 length; + uint16 frequency; + } trail; + + message { + uint64 id; + } @ctype(SpiceMsgDisplayInvalOne) inval_one; + + Empty inval_all; +}; + +enum32 audio_data_mode { + INVALID, + RAW, + CELT_0_5_1, +}; + +enum32 audio_fmt { + INVALID, + S16, +}; + +channel PlaybackChannel : BaseChannel { + server: + message { + uint32 time; + uint8 data[] @end @as_ptr(data_size); + } @ctype(SpiceMsgPlaybackPacket) data = 101; + + message { + uint32 time; + audio_data_mode mode; + uint8 data[] @end @as_ptr(data_size); + } mode; + + message { + uint32 channels; + audio_fmt format; + uint32 frequency; + uint32 time; + } start; + + Empty stop; +}; + +channel RecordChannel : BaseChannel { + server: + message { + uint32 channels; + audio_fmt format; + uint32 frequency; + } start = 101; + + Empty stop; + client: + message { + uint32 time; + uint8 data[] @end @nomarshal @as_ptr(data_size); + } @ctype(SpiceMsgcRecordPacket) data = 101; + + message { + uint32 time; + audio_data_mode mode; + uint8 data[] @end @as_ptr(data_size); + } mode; + + message { + uint32 time; + } start_mark; +}; + +protocol Spice { + MainChannel main = 1; + DisplayChannel display; + InputsChannel inputs; + CursorChannel cursor; + PlaybackChannel playback; + RecordChannel record; +}; From 2ef71169250a9b4830b576c0e93acb0259564f0d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 12:25:16 +0200 Subject: [PATCH 022/147] Remove minor markup in unstable protocol as we're resetting minor to 0 --- spice.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spice.proto b/spice.proto index dbd3dde..f722e7d 100644 --- a/spice.proto +++ b/spice.proto @@ -168,11 +168,11 @@ channel MainChannel : BaseChannel { uint16 sport; uint32 host_offset; uint32 host_size; - pubkey_type pub_key_type @minor(2); - uint32 pub_key_offset @minor(2); - uint32 pub_key_size @minor(2); + pubkey_type pub_key_type; + uint32 pub_key_offset; + uint32 pub_key_size; uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; - uint8 pub_key_data[pub_key_size] @minor(2) @end @ctype(uint8_t) @zero_terminated @nomarshal; + uint8 pub_key_data[pub_key_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; From 32dbd8a6a551345a829622ab92b356b6a3798839 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 13:37:25 +0200 Subject: [PATCH 023/147] Fix handling of @ptr32 network size --- python_modules/ptypes.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 101538c..9141539 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -426,6 +426,7 @@ class PointerType(Type): Type.__init__(self) self.name = None self.target_type = target_type + self.pointer_size = 8 def __str__(self): return "%s*" % (str(self.target_type)) @@ -434,8 +435,8 @@ class PointerType(Type): self.target_type = self.target_type.resolve() return self - def get_fixed_size(self): - return 8 # offsets are 64bit + def set_ptr_size(self, new_size): + self.pointer_size = new_size def is_fixed_nw_size(self): return True @@ -444,10 +445,13 @@ class PointerType(Type): return True def primitive_type(self): - return "uint64" + if self.pointer_size == 4: + return "uint32" + else: + return "uint64" def get_fixed_nw_size(self): - return 8 + return self.pointer_size def c_type(self): return "SPICE_ADDRESS" @@ -504,6 +508,8 @@ class Member(Containee): self.container = container self.member_type = self.member_type.resolve() self.member_type.register() + if self.has_attr("ptr32") and self.member_type.is_pointer(): + self.member_type.set_ptr_size(4) return self def is_primitive(self): From db5b8fa673d72c5726f4e358c04e9f1f97039a7f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 16:18:21 +0200 Subject: [PATCH 024/147] marshaller: Correctly determine if switches are fixed size Switches are fixed size only if all cases have the same size *and* it has a default case or all the valid cases are listed. --- python_modules/ptypes.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 9141539..2ee1789 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -630,14 +630,22 @@ class Switch(Containee): return True size = None + has_default = False for c in self.cases: + for v in c.values: + if v == None: + has_default = True if not c.member.is_fixed_nw_size(): return False if size == None: size = c.member.get_fixed_nw_size() elif size != c.member.get_fixed_nw_size(): return False - return True + # Fixed size if all elements listed, or has default + if has_default: + return True + key = self.container.lookup_member(self.variable) + return len(self.cases) == len(key.member_type.values) def is_extra_size(self): return self.has_end_attr() From 1d5d5272e28f8d12f5b93e4d8e5f85561eafda25 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 16:20:33 +0200 Subject: [PATCH 025/147] Make pointers 32bit in new protocol format --- python_modules/demarshal.py | 15 ++++++--------- python_modules/marshal.py | 4 ++-- python_modules/ptypes.py | 4 +++- spice_codegen.py | 4 ++++ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index a0697a9..0138ebe 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -184,7 +184,7 @@ def write_validate_struct_function(writer, struct): def write_validate_pointer_item(writer, container, item, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): if want_nw_size: - writer.assign(item.nw_size(), 8) + writer.assign(item.nw_size(), item.type.get_fixed_nw_size()) if want_mem_size or want_extra_size: target_type = item.type.target_type @@ -636,7 +636,7 @@ def write_switch_parser(writer, container, switch, dest, scope): if t.is_struct(): write_container_parser(writer, t, dest2) elif t.is_pointer(): - write_parse_pointer(writer, t, False, dest2, m.name, not m.has_attr("ptr32"), block) + write_parse_pointer(writer, t, False, dest2, m.name, block) elif t.is_primitive(): writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums @@ -724,12 +724,9 @@ def write_array_parser(writer, nelements, array, dest, scope): dest2.reuse_scope = array_scope write_container_parser(writer, element_type, dest2) -def write_parse_pointer(writer, t, at_end, dest, member_name, is_64bit, scope): +def write_parse_pointer(writer, t, at_end, dest, member_name, scope): target_type = t.target_type - if is_64bit: - writer.assign("ptr_info[n_ptr].offset", "consume_uint64(&in)") - else: - writer.assign("ptr_info[n_ptr].offset", "consume_uint32(&in)") + writer.assign("ptr_info[n_ptr].offset", "consume_%s(&in)" % t.primitive_type()) writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) if at_end: writer.assign("ptr_info[n_ptr].dest", "end") @@ -756,9 +753,9 @@ def write_member_parser(writer, container, member, dest, scope): if t.is_pointer(): if member.has_attr("nocopy"): writer.comment("Reuse data from network message").newline() - writer.assign(dest.get_ref(member.name), "(size_t)(message_start + consume_uint64(&in))") + writer.assign(dest.get_ref(member.name), "(size_t)(message_start + consume_%s(&in))" % t.primitive_type()) else: - write_parse_pointer(writer, t, member.has_end_attr(), dest, member.name, not member.has_attr("ptr32"), scope) + write_parse_pointer(writer, t, member.has_end_attr(), dest, member.name, scope) elif t.is_primitive(): if member.has_end_attr(): writer.statement("*(%s *)end = consume_%s(&in)" % (t.c_type(), t.primitive_type())) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index c5afd7c..ef1a47c 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -239,7 +239,7 @@ def write_switch_marshaller(writer, container, switch, src, scope): write_container_marshaller(writer, t, src2) elif t.is_pointer(): ptr_func = write_marshal_ptr_function(writer, t.target_type) - writer.assign("*%s_out" % (writer.out_prefix + m.name), "spice_marshaller_get_ptr_submarshaller(m, %s)" % ("0" if m.has_attr("ptr32") else "1")) + writer.assign("*%s_out" % (writer.out_prefix + m.name), "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if m.get_fixed_nw_size() == 8 else 0)) elif t.is_primitive(): if m.has_attr("zero"): writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) @@ -285,7 +285,7 @@ def write_member_marshaller(writer, container, member, src, scope): # else: # write_parse_pointer(writer, t, member.has_end_attr(), src, member.name, scope) ptr_func = write_marshal_ptr_function(writer, t.target_type) - writer.assign("*%s_out" % (writer.out_prefix + member.name), "spice_marshaller_get_ptr_submarshaller(m, %s)" % ("0" if member.has_attr("ptr32") else "1")) + writer.assign("*%s_out" % (writer.out_prefix + member.name), "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if member.get_fixed_nw_size() == 8 else 0)) elif t.is_primitive(): if member.has_attr("zero"): writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 2ee1789..f4126a5 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -4,6 +4,8 @@ import types _types_by_name = {} _types = [] +default_pointer_size = 4 + def type_exists(name): return _types_by_name.has_key(name) @@ -426,7 +428,7 @@ class PointerType(Type): Type.__init__(self) self.name = None self.target_type = target_type - self.pointer_size = 8 + self.pointer_size = default_pointer_size def __str__(self): return "%s*" % (str(self.target_type)) diff --git a/spice_codegen.py b/spice_codegen.py index 1d2314a..3a9989d 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -115,6 +115,8 @@ parser.add_option("-i", "--include", help="Include FILE in generated code") parser.add_option("--prefix", dest="prefix", help="set public symbol prefix", default="") +parser.add_option("--ptrsize", dest="ptrsize", + help="set default pointer size", default="4") (options, args) = parser.parse_args() @@ -124,6 +126,8 @@ if len(args) == 0: if len(args) == 1: parser.error("No destination file specified") +ptypes.default_pointer_size = int(options.ptrsize) + proto_file = args[0] dest_file = args[1] proto = spice_parser.parse(proto_file) From 31fd48e7737cc373b36a0ff6dbbd3fa07e2ff8f7 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 16:20:55 +0200 Subject: [PATCH 026/147] Don't send zero data item for clips with no data --- spice.proto | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/spice.proto b/spice.proto index f722e7d..240ed05 100644 --- a/spice.proto +++ b/spice.proto @@ -412,9 +412,7 @@ struct Path { struct Clip { clip_type type; switch (type) { - case NONE: - uint64 data @zero; - case RECTS: + case RECTS: ClipRects *data @outvar(cliprects); case PATH: Path *data @outvar(clippath); From baa0718cec0df2ff4a13d52929f323652ce11a1e Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 16:30:29 +0200 Subject: [PATCH 027/147] spice.proto: Don't use @fixedsize for Brush There is no reason to send the maximum size on the network, that is just a waste of space most of the time. --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index 240ed05..0d970da 100644 --- a/spice.proto +++ b/spice.proto @@ -529,7 +529,7 @@ struct Brush { uint32 color; case PATTERN: Pattern pattern; - } u @fixedsize; + } u; }; struct QMask { From fc30c38a9499c476c5506d5dd6fdaa64fb1808dd Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Jun 2010 16:35:09 +0200 Subject: [PATCH 028/147] spice.proto: No need for @ptr32 as that is now the default --- spice.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index 0d970da..199fa38 100644 --- a/spice.proto +++ b/spice.proto @@ -1052,8 +1052,8 @@ channel TunnelChannel : BaseChannel { uint32 id; uint32 group; uint32 port; - uint8 *name[cstring()] @ptr32 @nocopy; - uint8 *description[cstring()] @ptr32 @nocopy; + uint8 *name[cstring()] @nocopy; + uint8 *description[cstring()] @nocopy; switch (type) { case IPP: TunnelIpInfo ip @ctype(SpiceMsgTunnelIpInfo); From 7578be4a2398e0d00ddb6aefdeae0d77c32ea67d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 24 Jun 2010 14:30:00 +0200 Subject: [PATCH 029/147] Remove support for clip by path This is not supported currently anyway and was not generated before. --- spice.proto | 5 +---- spice1.proto | 11 ++++------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/spice.proto b/spice.proto index 199fa38..acb9488 100644 --- a/spice.proto +++ b/spice.proto @@ -253,8 +253,7 @@ channel MainChannel : BaseChannel { enum32 clip_type { NONE, - RECTS, - PATH, + RECTS }; flags32 path_flags { /* TODO: C enum names changes */ @@ -414,8 +413,6 @@ struct Clip { switch (type) { case RECTS: ClipRects *data @outvar(cliprects); - case PATH: - Path *data @outvar(clippath); } u @anon; }; diff --git a/spice1.proto b/spice1.proto index 0efe74b..dbe57b5 100644 --- a/spice1.proto +++ b/spice1.proto @@ -253,8 +253,7 @@ channel MainChannel : BaseChannel { enum32 clip_type { NONE, - RECTS, - PATH, + RECTS }; flags32 path_flags { /* TODO: C enum names changes */ @@ -382,12 +381,10 @@ struct Path { struct Clip { clip_type type; switch (type) { - case NONE: - uint64 data @zero; - case RECTS: + case RECTS: ClipRects *data @outvar(cliprects); - case PATH: - Path *data @outvar(clippath); + default: + uint64 data @zero; } u @anon; }; From 8b4ab9026db83bed561ca1286f9f40e6b8a8e7a3 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 24 Jun 2010 14:31:01 +0200 Subject: [PATCH 030/147] Use smaller size for enums and flags on the network This makes the protocol more compact. This was mainly done for the commonly used types. Some seldom used ones are still 32bit for future compatibility. --- spice.proto | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/spice.proto b/spice.proto index acb9488..d66445d 100644 --- a/spice.proto +++ b/spice.proto @@ -64,7 +64,7 @@ enum32 notify_visibility { HIGH, }; -flags32 mouse_mode { +flags16 mouse_mode { SERVER, CLIENT, }; @@ -251,27 +251,27 @@ channel MainChannel : BaseChannel { } @ctype(SpiceMsgcMainAgentTokens) agent_token; }; -enum32 clip_type { +enum8 clip_type { NONE, RECTS }; -flags32 path_flags { /* TODO: C enum names changes */ +flags8 path_flags { /* TODO: C enum names changes */ BEGIN = 0, END = 1, CLOSE = 3, BEZIER = 4, } @prefix(SPICE_PATH_); -enum32 video_codec_type { +enum8 video_codec_type { MJPEG = 1, }; -flags32 stream_flags { +flags8 stream_flags { TOP_DOWN = 0, }; -enum32 brush_type { +enum8 brush_type { NONE, SOLID, PATTERN, @@ -361,7 +361,7 @@ enum8 line_join { MITER, }; -flags16 string_flags { +flags8 string_flags { RASTER_A1, RASTER_A4, RASTER_A8, @@ -382,7 +382,7 @@ enum32 surface_fmt { 32_ARGB = 96 }; -flags16 alpha_flags { +flags8 alpha_flags { DEST_HAS_ALPHA, SRC_SURFACE_HAS_ALPHA }; @@ -787,13 +787,13 @@ channel DisplayChannel : BaseChannel { } init = 101; }; -flags32 keyboard_modifier_flags { +flags16 keyboard_modifier_flags { SCROLL_LOCK, NUM_LOCK, CAPS_LOCK }; -enum32 mouse_button { +enum8 mouse_button { INVALID, LEFT, MIDDLE, @@ -802,7 +802,7 @@ enum32 mouse_button { DOWN, }; -flags32 mouse_button_mask { +flags16 mouse_button_mask { LEFT, MIDDLE, RIGHT @@ -857,7 +857,7 @@ channel InputsChannel : BaseChannel { Empty mouse_motion_ack = 111; }; -enum16 cursor_type { +enum8 cursor_type { ALPHA, MONO, COLOR4, @@ -867,7 +867,7 @@ enum16 cursor_type { COLOR32, }; -flags32 cursor_flags { +flags16 cursor_flags { NONE, /* Means no cursor */ CACHE_ME, FROM_CACHE, @@ -924,13 +924,13 @@ channel CursorChannel : BaseChannel { Empty inval_all; }; -enum32 audio_data_mode { +enum16 audio_data_mode { INVALID, RAW, CELT_0_5_1, }; -enum32 audio_fmt { +enum16 audio_fmt { INVALID, S16, }; @@ -984,7 +984,7 @@ channel RecordChannel : BaseChannel { } start_mark; }; -enum32 tunnel_service_type { +enum16 tunnel_service_type { INVALID, GENERIC, IPP, From 302788f1474b4e6101ed68d77aa5bb562bafa23e Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 24 Jun 2010 14:33:06 +0200 Subject: [PATCH 031/147] spice.proto: surface_create.format is of type surface_fmt --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index d66445d..1aa2788 100644 --- a/spice.proto +++ b/spice.proto @@ -770,7 +770,7 @@ channel DisplayChannel : BaseChannel { uint32 surface_id; uint32 width; uint32 height; - uint32 format; + surface_fmt format; surface_flags flags; } @ctype(SpiceMsgSurfaceCreate) surface_create; From 144b2fbd75812b38d1c6086ca2561f47bc371898 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 28 Jun 2010 12:45:07 +0200 Subject: [PATCH 032/147] Fix build error due to member "SpiceMsgEmpty" same name as type --- python_modules/marshal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index ef1a47c..1eb3675 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -369,6 +369,8 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): writer.writeln("static SpiceMessageMarshallers marshallers = {NULL};").newline() for f in sorted(functions.keys()): member = f[len("spice_marshall_"):] + if not member.startswith("msg"): + member = "msg_" + member writer.assign("marshallers.%s" % member, f) writer.newline() From 435b1515a7555200d97fce2579088ff0b33949c8 Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Wed, 23 Jun 2010 08:53:04 +0200 Subject: [PATCH 033/147] qxl abi: parse QXLFill. Also adapt to tyops fix (s/rop_decriptor/rop_descriptor/). --- spice.proto | 2 +- spice1.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index 1aa2788..f2a1903 100644 --- a/spice.proto +++ b/spice.proto @@ -651,7 +651,7 @@ channel DisplayChannel : BaseChannel { DisplayBase base; struct Fill { Brush brush @outvar(brush); - uint16 rop_decriptor; + uint16 rop_descriptor; QMask mask @outvar(mask); } data; } draw_fill = 302; diff --git a/spice1.proto b/spice1.proto index dbe57b5..90a2c30 100644 --- a/spice1.proto +++ b/spice1.proto @@ -600,7 +600,7 @@ channel DisplayChannel : BaseChannel { DisplayBase base; struct Fill { Brush brush @outvar(brush); - uint16 rop_decriptor; + uint16 rop_descriptor; QMask mask @outvar(mask); } data; } draw_fill = 302; From 54fc0642e14a70891dc4de1e64001eae5dc2702b Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Wed, 23 Jun 2010 09:49:35 +0200 Subject: [PATCH 034/147] qxl abi: parse QXLOpaque. Also adapt to tyops fix (s/rop_decriptor/rop_descriptor/). --- spice.proto | 2 +- spice1.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index f2a1903..38493a4 100644 --- a/spice.proto +++ b/spice.proto @@ -662,7 +662,7 @@ channel DisplayChannel : BaseChannel { Image *src_bitmap; Rect src_area; Brush brush; - ropd rop_decriptor; + ropd rop_descriptor; image_scale_mode scale_mode; QMask mask @outvar(mask); } data; diff --git a/spice1.proto b/spice1.proto index 90a2c30..a233ffc 100644 --- a/spice1.proto +++ b/spice1.proto @@ -611,7 +611,7 @@ channel DisplayChannel : BaseChannel { Image *src_bitmap; Rect src_area; Brush brush; - ropd rop_decriptor; + ropd rop_descriptor; image_scale_mode scale_mode; QMask mask @outvar(mask); } data; From f2d7fbdb19cd9758346316e63bcbd87b7dedd5ff Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Wed, 23 Jun 2010 14:46:23 +0200 Subject: [PATCH 035/147] qxl abi: parse QXLCopy + QXLBlend. Also adapt to tyops fix (s/rop_decriptor/rop_descriptor/). --- spice.proto | 4 ++-- spice1.proto | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spice.proto b/spice.proto index 38493a4..e1062eb 100644 --- a/spice.proto +++ b/spice.proto @@ -673,7 +673,7 @@ channel DisplayChannel : BaseChannel { struct Copy { Image *src_bitmap; Rect src_area; - ropd rop_decriptor; + ropd rop_descriptor; image_scale_mode scale_mode; QMask mask @outvar(mask); } data; @@ -684,7 +684,7 @@ channel DisplayChannel : BaseChannel { struct Blend { Image *src_bitmap; Rect src_area; - ropd rop_decriptor; + ropd rop_descriptor; image_scale_mode scale_mode; QMask mask @outvar(mask); } @ctype(SpiceCopy) data; diff --git a/spice1.proto b/spice1.proto index a233ffc..98ffb49 100644 --- a/spice1.proto +++ b/spice1.proto @@ -622,7 +622,7 @@ channel DisplayChannel : BaseChannel { struct Copy { Image *src_bitmap; Rect src_area; - ropd rop_decriptor; + ropd rop_descriptor; image_scale_mode scale_mode; QMask mask @outvar(mask); } data; @@ -633,7 +633,7 @@ channel DisplayChannel : BaseChannel { struct Blend { Image *src_bitmap; Rect src_area; - ropd rop_decriptor; + ropd rop_descriptor; image_scale_mode scale_mode; QMask mask @outvar(mask); } @ctype(SpiceCopy) data; From dcaba909d3d68f7dcb3077de083724daf4b6fd29 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 29 Jun 2010 18:13:07 +0200 Subject: [PATCH 036/147] demarshaller: Don't parse @zero members These just write zeros at the right place in the network protocol typically for old back-compat things. We don't want to read these back in. --- python_modules/demarshal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 0138ebe..d8b8ea9 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -638,7 +638,8 @@ def write_switch_parser(writer, container, switch, dest, scope): elif t.is_pointer(): write_parse_pointer(writer, t, False, dest2, m.name, block) elif t.is_primitive(): - writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) + if not m.has_attr("zero"): + writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums elif t.is_array(): nelements = read_array_len(writer, m.name, t, dest, block) @@ -757,6 +758,8 @@ def write_member_parser(writer, container, member, dest, scope): else: write_parse_pointer(writer, t, member.has_end_attr(), dest, member.name, scope) elif t.is_primitive(): + if member.has_attr("zero"): + pass if member.has_end_attr(): writer.statement("*(%s *)end = consume_%s(&in)" % (t.c_type(), t.primitive_type())) writer.increment("end", t.sizeof()) From 13f8149daf9f38f76ec843599850c7fdea15f545 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 29 Jun 2010 18:14:49 +0200 Subject: [PATCH 037/147] demarshaller: Support @c_ptr attributes for pointers A @c_ptr pointer is stored in memory as a real pointer rather than a SPICE_ADDRESS. This is a temporary thing that will be removed again when all SPICE_ADDRESSes have been converted to real pointer. --- python_modules/demarshal.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index d8b8ea9..faaf862 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -50,8 +50,9 @@ def write_parser_helpers(writer): writer.begin_block("struct PointerInfo") writer.variable_def("uint64_t", "offset") writer.variable_def("parse_func_t", "parse") - writer.variable_def("SPICE_ADDRESS *", "dest") + writer.variable_def("void *", "dest") writer.variable_def("uint32_t", "nelements") + writer.variable_def("int", "is_ptr") writer.end_block(semicolon=True) def write_read_primitive(writer, start, container, name, scope): @@ -636,7 +637,7 @@ def write_switch_parser(writer, container, switch, dest, scope): if t.is_struct(): write_container_parser(writer, t, dest2) elif t.is_pointer(): - write_parse_pointer(writer, t, False, dest2, m.name, block) + write_parse_pointer(writer, t, False, m.has_attr("c_ptr"), dest2, m.name, block) elif t.is_primitive(): if not m.has_attr("zero"): writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) @@ -725,15 +726,16 @@ def write_array_parser(writer, nelements, array, dest, scope): dest2.reuse_scope = array_scope write_container_parser(writer, element_type, dest2) -def write_parse_pointer(writer, t, at_end, dest, member_name, scope): +def write_parse_pointer(writer, t, at_end, as_c_ptr, dest, member_name, scope): target_type = t.target_type writer.assign("ptr_info[n_ptr].offset", "consume_%s(&in)" % t.primitive_type()) writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) if at_end: writer.assign("ptr_info[n_ptr].dest", "end") - writer.increment("end", "sizeof(SPICE_ADDRESS)"); + writer.increment("end", "sizeof(void *)" if as_c_ptr else "sizeof(SPICE_ADDRESS)"); else: writer.assign("ptr_info[n_ptr].dest", "&%s" % dest.get_ref(member_name)) + writer.assign("ptr_info[n_ptr].is_ptr", "1" if as_c_ptr else "0") if target_type.is_array(): nelements = read_array_len(writer, member_name, target_type, dest, scope) writer.assign("ptr_info[n_ptr].nelements", nelements) @@ -756,7 +758,7 @@ def write_member_parser(writer, container, member, dest, scope): writer.comment("Reuse data from network message").newline() writer.assign(dest.get_ref(member.name), "(size_t)(message_start + consume_%s(&in))" % t.primitive_type()) else: - write_parse_pointer(writer, t, member.has_end_attr(), dest, member.name, scope) + write_parse_pointer(writer, t, member.has_end_attr(), member.has_attr("c_ptr"), dest, member.name, scope) elif t.is_primitive(): if member.has_attr("zero"): pass @@ -817,13 +819,20 @@ def write_ptr_info_check(writer): with writer.for_loop(index, "n_ptr") as scope: offset = "ptr_info[%s].offset" % index function = "ptr_info[%s].parse" % index + is_ptr = "ptr_info[%s].is_ptr" % index dest = "ptr_info[%s].dest" % index with writer.if_block("%s == 0" % offset, newline=False): - writer.assign("*%s" % dest, "0") + with writer.if_block("%s == 0" % is_ptr, newline=False): + writer.assign("*(void **)(%s)" % dest, "NULL") + with writer.block(" else"): + writer.assign("*(SPICE_ADDRESS *)(%s)" % dest, "0") with writer.block(" else"): writer.comment("Align to 32 bit").newline() writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") - writer.assign("*%s" % dest, "(size_t)end") + with writer.if_block("%s == 0" % is_ptr, newline=False): + writer.assign("*(void **)(%s)" % dest, "(void *)end") + with writer.block(" else"): + writer.assign("*(SPICE_ADDRESS *)(%s)" % dest, "(SPICE_ADDRESS)end") writer.assign("end", "%s(message_start, message_end, end, &ptr_info[%s], minor)" % (function, index)) writer.error_check("end == NULL") writer.newline() From 140cf2aa795abf20482eb6684f15aef5d187ebca Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 29 Jun 2010 18:17:35 +0200 Subject: [PATCH 038/147] Update client and protocol to support the new SpiceClipRects --- spice.proto | 2 +- spice1.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index e1062eb..c399150 100644 --- a/spice.proto +++ b/spice.proto @@ -412,7 +412,7 @@ struct Clip { clip_type type; switch (type) { case RECTS: - ClipRects *data @outvar(cliprects); + ClipRects *rects @outvar(cliprects) @c_ptr; } u @anon; }; diff --git a/spice1.proto b/spice1.proto index 98ffb49..75749af 100644 --- a/spice1.proto +++ b/spice1.proto @@ -382,7 +382,7 @@ struct Clip { clip_type type; switch (type) { case RECTS: - ClipRects *data @outvar(cliprects); + ClipRects *rects @outvar(cliprects) @c_ptr; default: uint64 data @zero; } u @anon; From e42c910b5cb5e6de734064fb57832a6e73e34092 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 29 Jun 2010 21:42:59 +0200 Subject: [PATCH 039/147] Store SpicePath segment count rather than size Internally and in the network protocol (for the new version) we now store the actual number of segments rather than the size of the full segments array in bytes. This change consists of multiple changes to handle this: * Make the qxl parser calculate num_segments * Make the canvas stroke code handle the new SpicePath layout. * Fix up is_equal_path in red_worker.c for the new layout * replace multiple calls to spice_marshall_PathSegment with a single spice_marshall_Path call * Make the byte_size() array size handling do the conversion from network size to number of elements when marshalling/demarshalling. * Update the current spice protocol to send the segment count rather than the size * Update the old spice protocol to use the new byte_size functionallity to calculate the size sent and the number of elements recieved --- python_modules/demarshal.py | 16 ++++++++++++++-- python_modules/marshal.py | 27 ++++++++++++++++++--------- python_modules/spice_parser.py | 2 +- spice.proto | 4 ++-- spice1.proto | 4 ++-- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index faaf862..48b4e73 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -604,7 +604,7 @@ def read_array_len(writer, prefix, array, dest, scope, handles_bytes = False): elif array.is_bytes_length(): if not handles_bytes: raise NotImplementedError("handling of bytes() not supported here yet") - writer.assign(nelements, dest.get_ref(array.size[1])) + writer.assign(nelements, array.size[1]) else: raise NotImplementedError("TODO array size type not handled yet") return nelements @@ -714,10 +714,15 @@ def write_array_parser(writer, nelements, array, dest, scope): writer.increment("end", nelements) else: if is_byte_size: + real_nelements = nelements[:-len("nbytes")] + "nelements" scope.variable_def("uint8_t *", "array_end") + scope.variable_def("uint32_t", real_nelements) writer.assign("array_end", "end + %s" % nelements) + writer.assign(real_nelements, 0) with writer.index(no_block = is_byte_size) as index: with writer.while_loop("end < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: + if is_byte_size: + writer.increment(real_nelements, 1) if element_type.is_primitive(): writer.statement("*(%s *)end = consume_%s(&in)" % (element_type.c_type(), element_type.primitive_type())) writer.increment("end", element_type.sizeof()) @@ -725,6 +730,8 @@ def write_array_parser(writer, nelements, array, dest, scope): dest2 = dest.child_at_end(writer, element_type) dest2.reuse_scope = array_scope write_container_parser(writer, element_type, dest2) + if is_byte_size: + writer.assign(dest.get_ref(array.size[2]), real_nelements) def write_parse_pointer(writer, t, at_end, as_c_ptr, dest, member_name, scope): target_type = t.target_type @@ -766,7 +773,12 @@ def write_member_parser(writer, container, member, dest, scope): writer.statement("*(%s *)end = consume_%s(&in)" % (t.c_type(), t.primitive_type())) writer.increment("end", t.sizeof()) else: - writer.assign(dest.get_ref(member.name), "consume_%s(&in)" % (t.primitive_type())) + if member.has_attr("bytes_count"): + scope.variable_def("uint32_t", member.name); + dest_var = member.name + else: + dest_var = dest.get_ref(member.name) + writer.assign(dest_var, "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums elif t.is_array(): nelements = read_array_len(writer, member.name, t, dest, scope, handles_bytes = True) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 1eb3675..250147f 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -165,7 +165,7 @@ def get_array_size(array, container_src): else: return "(((%s * %s + 7) / 8 ) * %s)" % (bpp, width_v, rows_v) elif array.is_bytes_length(): - return container_src.get_ref(array.size[1]) + return container_src.get_ref(array.size[2]) else: raise NotImplementedError("TODO array size type not handled yet") @@ -179,20 +179,18 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): nelements = get_array_size(array, container_src) is_byte_size = array.is_bytes_length() - if is_byte_size: - element = "%s__bytes" % member.name - else: - element = "%s__element" % member.name + element = "%s__element" % member.name if not at_end: writer.assign(element, container_src.get_ref(member.name)) if is_byte_size: - scope.variable_def("size_t", "array_end") - writer.assign("array_end", "spice_marshaller_get_size(m) + %s" % nelements) + size_start_var = "%s__size_start" % member.name + scope.variable_def("size_t", size_start_var) + writer.assign(size_start_var, "spice_marshaller_get_size(m)") - with writer.index(no_block = is_byte_size) as index: - with writer.while_loop("spice_marshaller_get_size(m) < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: + with writer.index() as index: + with writer.for_loop(index, nelements) as array_scope: array_scope.variable_def(element_type.c_type() + " *", element) if at_end: writer.assign(element, "(%s *)end" % element_type.c_type()) @@ -210,6 +208,12 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): if not at_end: writer.statement("%s++" % element) + if is_byte_size: + size_var = member.container.lookup_member(array.size[1]) + size_var_type = size_var.member_type + var = "%s__ref" % array.size[1] + writer.statement("spice_marshaller_set_%s(m, %s, spice_marshaller_get_size(m) - %s)" % (size_var_type.primitive_type(), var, size_start_var)) + def write_switch_marshaller(writer, container, switch, src, scope): var = container.lookup_member(switch.variable) var_type = var.member_type @@ -289,6 +293,11 @@ def write_member_marshaller(writer, container, member, src, scope): elif t.is_primitive(): if member.has_attr("zero"): writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) + if member.has_attr("bytes_count"): + var = "%s__ref" % member.name + scope.variable_def("void *", var) + writer.statement("%s = spice_marshaller_add_%s(m, %s)" % (var, t.primitive_type(), 0)) + elif member.has_end_attr(): writer.statement("spice_marshaller_add_%s(m, *(%s_t *)end)" % (t.primitive_type(), t.primitive_type())) writer.increment("end", t.sizeof()) diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index 65916b3..61ef458 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -87,7 +87,7 @@ def SPICE_BNF(): attribute = Group(Combine ("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen)) attributes = Group(ZeroOrMore(attribute)) arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen) - arraySizeSpecBytes = Group(bytes_ + lparen + identifier + rparen) + arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma + identifier + rparen) arraySizeSpecCString = Group(cstring_ + lparen + rparen) arraySizeSpec = lbrack + Optional(identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^arraySizeSpecCString, default="") + rbrack variableDef = Group(typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi) \ diff --git a/spice.proto b/spice.proto index c399150..dedf950 100644 --- a/spice.proto +++ b/spice.proto @@ -404,8 +404,8 @@ struct PathSegment { } @ctype(SpicePathSeg); struct Path { - uint32 size; - PathSegment segments[bytes(size)] @end; + uint32 num_segments; + PathSegment segments[num_segments] @end; }; struct Clip { diff --git a/spice1.proto b/spice1.proto index 75749af..0150de2 100644 --- a/spice1.proto +++ b/spice1.proto @@ -374,8 +374,8 @@ struct PathSegment { } @ctype(SpicePathSeg); struct Path { - uint32 size; - PathSegment segments[bytes(size)] @end; + uint32 segments_size @bytes_count; + PathSegment segments[bytes(segments_size, num_segments)] @end; }; struct Clip { From 02a429e46e526bbdefc7c2ca44fc585f2a90826e Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 30 Jun 2010 13:24:59 +0200 Subject: [PATCH 040/147] Support @marshall to automatically marshall pointers --- python_modules/marshal.py | 39 ++++++++++++++++++++++++--------------- python_modules/ptypes.py | 29 ++++++++++++++++------------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 250147f..da9b0d8 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -104,10 +104,10 @@ def write_marshal_ptr_function(writer, target_type): writer.set_is_generated("marshaller", marshal_function) - names = target_type.get_pointer_names() + names = target_type.get_pointer_names(False) names_args = "" if len(names) > 0: - n = map(lambda name: ", SpiceMarshaller **%s" % name, names) + n = map(lambda name: ", SpiceMarshaller **%s_out" % name, names) names_args = "".join(n) header = writer.header @@ -120,9 +120,10 @@ def write_marshal_ptr_function(writer, target_type): scope = writer.function(marshal_function, "void *", "SpiceMarshaller *m, %s *ptr" % target_type.c_type() + names_args) header.writeln("void *" + marshal_function + "(SpiceMarshaller *m, %s *msg" % target_type.c_type() + names_args + ");") scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") + scope.variable_def("SPICE_GNUC_UNUSED SpiceMarshaller *", "m2") for n in names: - writer.assign("*%s" % n, "NULL") + writer.assign("*%s_out" % n, "NULL") writer.newline() writer.assign("end", "(uint8_t *)(ptr+1)") @@ -214,6 +215,20 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): var = "%s__ref" % array.size[1] writer.statement("spice_marshaller_set_%s(m, %s, spice_marshaller_get_size(m) - %s)" % (size_var_type.primitive_type(), var, size_start_var)) +def write_pointer_marshaller(writer, member, src): + t = member.member_type + ptr_func = write_marshal_ptr_function(writer, t.target_type) + submarshaller = "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if member.get_fixed_nw_size() == 8 else 0) + if member.has_attr("marshall"): + writer.assign("m2", submarshaller) + if member.has_attr("nonnull"): + writer.statement("%s(m2, %s)" % (ptr_func, src.get_ref(member.name))) + else: + with writer.if_block("%s != NULL" % src.get_ref(member.name)) as block: + writer.statement("%s(m2, %s)" % (ptr_func, src.get_ref(member.name))) + else: + writer.assign("*%s_out" % (writer.out_prefix + member.name), submarshaller) + def write_switch_marshaller(writer, container, switch, src, scope): var = container.lookup_member(switch.variable) var_type = var.member_type @@ -242,8 +257,7 @@ def write_switch_marshaller(writer, container, switch, src, scope): if t.is_struct(): write_container_marshaller(writer, t, src2) elif t.is_pointer(): - ptr_func = write_marshal_ptr_function(writer, t.target_type) - writer.assign("*%s_out" % (writer.out_prefix + m.name), "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if m.get_fixed_nw_size() == 8 else 0)) + write_pointer_marshaller(writer, m, src2) elif t.is_primitive(): if m.has_attr("zero"): writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) @@ -283,13 +297,7 @@ def write_member_marshaller(writer, container, member, src, scope): t = member.member_type if t.is_pointer(): -# if member.has_attr("nocopy"): -# writer.comment("Reuse data from network message").newline() -# writer.assign(src.get_ref(member.name), "(size_t)(message_start + consume_uint64(&in))") -# else: -# write_parse_pointer(writer, t, member.has_end_attr(), src, member.name, scope) - ptr_func = write_marshal_ptr_function(writer, t.target_type) - writer.assign("*%s_out" % (writer.out_prefix + member.name), "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if member.get_fixed_nw_size() == 8 else 0)) + write_pointer_marshaller(writer, member, src) elif t.is_primitive(): if member.has_attr("zero"): writer.statement("spice_marshaller_add_%s(m, 0)" % (t.primitive_type())) @@ -329,10 +337,10 @@ def write_message_marshaller(writer, message, is_server, private): return function_name writer.set_is_generated("marshaller", function_name) - names = message.get_pointer_names() + names = message.get_pointer_names(False) names_args = "" if len(names) > 0: - n = map(lambda name: ", SpiceMarshaller **%s" % name, names) + n = map(lambda name: ", SpiceMarshaller **%s_out" % name, names) names_args = "".join(n) if not private: @@ -342,9 +350,10 @@ def write_message_marshaller(writer, message, is_server, private): "static void" if private else "void", "SpiceMarshaller *m, %s *msg" % message.c_type() + names_args) scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") + scope.variable_def("SPICE_GNUC_UNUSED SpiceMarshaller *", "m2") for n in names: - writer.assign("*%s" % n, "NULL") + writer.assign("*%s_out" % n, "NULL") src = RootMarshallingSource(None, message.c_type(), message.sizeof(), "msg") src.reuse_scope = scope diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index f4126a5..bef5432 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -94,7 +94,7 @@ class Type: def get_num_pointers(self): return 0 - def get_pointer_names(self): + def get_pointer_names(self, marshalled): return [] def sizeof(self): @@ -205,8 +205,8 @@ class TypeAlias(Type): def get_num_pointers(self): return self.the_type.get_num_pointers() - def get_pointer_names(self): - return self.the_type.get_pointer_names() + def get_pointer_names(self, marshalled): + return self.the_type.get_pointer_names(marshalled) def c_type(self): if self.has_attr("ctype"): @@ -408,7 +408,7 @@ class ArrayType(Type): return element_count * self.size raise Exception, "Pointers in dynamic arrays not supported" - def get_pointer_names(self): + def get_pointer_names(self, marshalled): element_count = self.element_type.get_num_pointers() if element_count == 0: return [] @@ -554,11 +554,14 @@ class Member(Containee): def get_num_pointers(self): return self.member_type.get_num_pointers() - def get_pointer_names(self): + def get_pointer_names(self, marshalled): if self.member_type.is_pointer(): - names = [self.name + "_out"] + if self.has_attr("marshall") == marshalled: + names = [self.name] + else: + names = [] else: - names = self.member_type.get_pointer_names() + names = self.member_type.get_pointer_names(marshalled) if self.has_attr("outvar"): prefix = self.attributes["outvar"][0] names = map(lambda name: prefix + "_" + name, names) @@ -592,8 +595,8 @@ class SwitchCase: def get_num_pointers(self): return self.member.get_num_pointers() - def get_pointer_names(self): - return self.member.get_pointer_names() + def get_pointer_names(self, marshalled): + return self.member.get_pointer_names(marshalled) class Switch(Containee): def __init__(self, variable, cases, name, attribute_list): @@ -684,10 +687,10 @@ class Switch(Containee): count = max(count, c.get_num_pointers()) return count - def get_pointer_names(self): + def get_pointer_names(self, marshalled): names = [] for c in self.cases: - names = names + c.get_pointer_names() + names = names + c.get_pointer_names(marshalled) return names class ContainerType(Type): @@ -736,10 +739,10 @@ class ContainerType(Type): count = count + m.get_num_pointers() return count - def get_pointer_names(self): + def get_pointer_names(self, marshalled): names = [] for m in self.members: - names = names + m.get_pointer_names() + names = names + m.get_pointer_names(marshalled) return names def has_pointer(self): From e191e472384231a5db063f1a1c628c0b20b5ba7a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 30 Jun 2010 14:05:04 +0200 Subject: [PATCH 041/147] Automatically marshall SpicePath --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index dedf950..0773f0a 100644 --- a/spice.proto +++ b/spice.proto @@ -726,7 +726,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Stroke { - Path *path; + Path *path @c_ptr @marshall @nonnull; LineAttr attr; Brush brush; uint16 fore_mode; From 68e29d0d1929366d267e25581fa920f3d30cc0a8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 30 Jun 2010 14:05:14 +0200 Subject: [PATCH 042/147] Automatically marshall SpiceClipRects --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index 0773f0a..67acb67 100644 --- a/spice.proto +++ b/spice.proto @@ -412,7 +412,7 @@ struct Clip { clip_type type; switch (type) { case RECTS: - ClipRects *rects @outvar(cliprects) @c_ptr; + ClipRects *rects @outvar(cliprects) @c_ptr @marshall @nonnull; } u @anon; }; From 6d38c4817f751f5f47dd1d22ba8319675bb57295 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 30 Jun 2010 16:49:50 +0200 Subject: [PATCH 043/147] Simplify SpiceLineAttr by removing unsed stuff Also in new protocol don't send style data if not needed. --- python_modules/demarshal.py | 11 +++++++---- python_modules/ptypes.py | 15 ++++++++++++++- spice.proto | 28 +++++++++------------------- spice1.proto | 10 +++++----- 4 files changed, 35 insertions(+), 29 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 48b4e73..b819513 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -62,7 +62,8 @@ def write_read_primitive(writer, start, container, name, scope): writer.error_check("pos + %s > message_end" % m.member_type.get_fixed_nw_size()) var = "%s__value" % (name) - scope.variable_def(m.member_type.c_type(), var) + if not scope.variable_defined(var): + scope.variable_def(m.member_type.c_type(), var) writer.assign(var, "read_%s(pos)" % (m.member_type.primitive_type())) return var @@ -639,7 +640,9 @@ def write_switch_parser(writer, container, switch, dest, scope): elif t.is_pointer(): write_parse_pointer(writer, t, False, m.has_attr("c_ptr"), dest2, m.name, block) elif t.is_primitive(): - if not m.has_attr("zero"): + if m.has_attr("zero"): + writer.statement("consume_%s(&in)" % (t.primitive_type())) + else: writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums elif t.is_array(): @@ -768,8 +771,8 @@ def write_member_parser(writer, container, member, dest, scope): write_parse_pointer(writer, t, member.has_end_attr(), member.has_attr("c_ptr"), dest, member.name, scope) elif t.is_primitive(): if member.has_attr("zero"): - pass - if member.has_end_attr(): + writer.statement("consume_%s(&in)" % t.primitive_type()) + elif member.has_end_attr(): writer.statement("*(%s *)end = consume_%s(&in)" % (t.c_type(), t.primitive_type())) writer.increment("end", t.sizeof()) else: diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index bef5432..b7bcac9 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -610,6 +610,12 @@ class Switch(Containee): def is_switch(self): return True + def lookup_case_member(self, name): + for c in self.cases: + if c.member.name == name: + return c.member + return None + def has_switch_member(self, member): for c in self.cases: if c.member == member: @@ -767,7 +773,14 @@ class ContainerType(Type): return str(fixed) def lookup_member(self, name): - return self.members_by_name[name] + if self.members_by_name.has_key(name): + return self.members_by_name[name] + for m in self.members: + if m.is_switch(): + member = m.lookup_case_member(name) + if member: + return member + raise Exception, "No member called %s found" % name class StructType(ContainerType): def __init__(self, name, members, attribute_list): diff --git a/spice.proto b/spice.proto index 67acb67..7528208 100644 --- a/spice.proto +++ b/spice.proto @@ -131,7 +131,7 @@ channel BaseChannel { uint32 what; /* error_code/warn_code/info_code */ uint32 message_len; uint8 message[message_len] @end @nomarshal; - uint8 zero @end @ctype(uint8_t) @zero @nomarshal; + uint8 zero @end @ctype(uint8_t) @nomarshal; } notify; client: @@ -349,18 +349,6 @@ flags8 line_flags { START_WITH_GAP = 2, }; -enum8 line_cap { - ROUND, - SQUARE, - BUTT, -}; - -enum8 line_join { - ROUND, - BEVEL, - MITER, -}; - flags8 string_flags { RASTER_A1, RASTER_A4, @@ -537,12 +525,14 @@ struct QMask { struct LineAttr { line_flags flags; - line_join join_style; - line_cap end_style; - uint8 style_nseg; - fixed28_4 width; - fixed28_4 miter_limit; - fixed28_4 *style[style_nseg]; + switch (flags) { + case STYLED: + uint8 style_nseg; + } u1 @anon; + switch (flags) { + case STYLED: + fixed28_4 *style[style_nseg]; + } u2 @anon; }; struct RasterGlyphA1 { diff --git a/spice1.proto b/spice1.proto index 0150de2..c9a18d9 100644 --- a/spice1.proto +++ b/spice1.proto @@ -131,7 +131,7 @@ channel BaseChannel { uint32 what; /* error_code/warn_code/info_code */ uint32 message_len; uint8 message[message_len] @end @nomarshal; - uint8 zero @end @ctype(uint8_t) @zero @nomarshal; + uint8 zero @end @ctype(uint8_t) @nomarshal; } notify; client: @@ -485,11 +485,11 @@ struct QMask { struct LineAttr { line_flags flags; - line_join join_style; - line_cap end_style; + line_join join_style @zero; + line_cap end_style @zero; uint8 style_nseg; - fixed28_4 width; - fixed28_4 miter_limit; + fixed28_4 width @zero; + fixed28_4 miter_limit @zero; fixed28_4 *style[style_nseg]; }; From 8ec1247dbe40ca7d0678e95c45f0d3a7b8908b77 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 1 Jul 2010 16:45:04 +0200 Subject: [PATCH 044/147] Fix 32bit failure in demarshaller Due to a typo we always read offsets as pointers, never as SPICE_ADDRESS. --- python_modules/demarshal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index b819513..023c3f6 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -837,14 +837,14 @@ def write_ptr_info_check(writer): is_ptr = "ptr_info[%s].is_ptr" % index dest = "ptr_info[%s].dest" % index with writer.if_block("%s == 0" % offset, newline=False): - with writer.if_block("%s == 0" % is_ptr, newline=False): + with writer.if_block(is_ptr, newline=False): writer.assign("*(void **)(%s)" % dest, "NULL") with writer.block(" else"): writer.assign("*(SPICE_ADDRESS *)(%s)" % dest, "0") with writer.block(" else"): writer.comment("Align to 32 bit").newline() writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") - with writer.if_block("%s == 0" % is_ptr, newline=False): + with writer.if_block(is_ptr, newline=False): writer.assign("*(void **)(%s)" % dest, "(void *)end") with writer.block(" else"): writer.assign("*(SPICE_ADDRESS *)(%s)" % dest, "(SPICE_ADDRESS)end") From 6228ae633e58f484f0da5cc20dcfbf42ead4859b Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Wed, 30 Jun 2010 22:19:12 +0200 Subject: [PATCH 045/147] Properly parse and marshall SpiceString --- python_modules/demarshal.py | 44 +++++++++++++++++++++++++++++++------ python_modules/marshal.py | 20 ++++++++++++++--- python_modules/ptypes.py | 8 ++++++- spice.proto | 10 ++++----- spice1.proto | 8 +++---- 5 files changed, 70 insertions(+), 20 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 023c3f6..4d3e79b 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -125,7 +125,7 @@ def write_validate_switch_member(writer, container, switch_member, scope, parent item.subprefix = item.prefix + "_" + m.name item.non_null = c.member.has_attr("nonnull") sub_want_extra_size = want_extra_size - if sub_want_extra_size and not m.contains_extra_size(): + if sub_want_extra_size and not m.contains_extra_size() and not m.is_extra_size(): writer.assign(item.extra_size(), 0) sub_want_extra_size = False @@ -309,10 +309,13 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star if element_type.is_fixed_sizeof() and want_mem_size and not is_byte_size: # TODO: Overflow check the multiplication - writer.assign(mem_size, "%s * %s" % (element_type.sizeof(), nelements)) + if array.ptr_array: + writer.assign(mem_size, "sizeof(void *) + SPICE_ALIGN(%s * %s, 4)" % (element_type.sizeof(), nelements)) + else: + writer.assign(mem_size, "%s * %s" % (element_type.sizeof(), nelements)) want_mem_size = False - if not element_type.contains_extra_size() and want_extra_size: + if not element_type.contains_extra_size() and not array.is_extra_size() and want_extra_size: writer.assign(extra_size, 0) want_extra_size = False @@ -329,14 +332,24 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star element_nw_size = element_item.nw_size() element_mem_size = element_item.mem_size() + element_extra_size = element_item.extra_size() scope.variable_def("uint32_t", element_nw_size) scope.variable_def("uint32_t", element_mem_size) + want_is_extra_size = False + want_element_mem_size = want_mem_size + if want_extra_size: + if array.is_extra_size(): + want_is_extra_size = True + want_extra_size = False + want_element_mem_size = True + else: + scope.variable_def("uint32_t", element_extra_size) if want_nw_size: writer.assign(nw_size, 0) if want_mem_size: writer.assign(mem_size, 0) - if want_extra_size: + if want_extra_size or want_is_extra_size: writer.assign(extra_size, 0) want_element_nw_size = want_nw_size @@ -352,13 +365,19 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star with writer.index(no_block = is_byte_size) as index: with writer.while_loop("%s < %s" % (start2, start2_end) ) if is_byte_size else writer.for_loop(index, nelements) as scope: write_validate_item(writer, container, element_item, scope, parent_scope, start2, - want_element_nw_size, want_mem_size, want_extra_size) + want_element_nw_size, want_element_mem_size, want_extra_size) if want_nw_size: writer.increment(nw_size, element_nw_size) if want_mem_size: - writer.increment(mem_size, element_mem_size) - if want_extra_size: + if not array.is_extra_size(): + writer.increment(mem_size, element_mem_size) + if want_is_extra_size: + if array.ptr_array: + writer.increment(extra_size, "sizeof(void *) + SPICE_ALIGN(%s, 4)" % element_mem_size) + else: + writer.increment(extra_size, "%s + %s" % (element_mem_size, element_extra_size)) + elif want_extra_size: writer.increment(extra_size, element_extra_size) writer.increment(start2, start_increment) @@ -722,8 +741,16 @@ def write_array_parser(writer, nelements, array, dest, scope): scope.variable_def("uint32_t", real_nelements) writer.assign("array_end", "end + %s" % nelements) writer.assign(real_nelements, 0) + if array.ptr_array: + scope.variable_def("void **", "ptr_array") + scope.variable_def("int", "ptr_array_index") + writer.assign("ptr_array_index", 0) + writer.assign("ptr_array", "(void **)end") + writer.increment("end", "sizeof(void *) * %s" % nelements) with writer.index(no_block = is_byte_size) as index: with writer.while_loop("end < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: + if array.ptr_array: + writer.statement("ptr_array[ptr_array_index++] = end") if is_byte_size: writer.increment(real_nelements, 1) if element_type.is_primitive(): @@ -733,6 +760,9 @@ def write_array_parser(writer, nelements, array, dest, scope): dest2 = dest.child_at_end(writer, element_type) dest2.reuse_scope = array_scope write_container_parser(writer, element_type, dest2) + if array.ptr_array: + writer.comment("Align ptr_array element to 4 bytes").newline() + writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") if is_byte_size: writer.assign(dest.get_ref(array.size[2]), real_nelements) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index da9b0d8..95413fc 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -50,6 +50,7 @@ class RootMarshallingSource(MarshallingSource): self.c_type = c_type self.sizeof = sizeof self.pointer = pointer # None == at "end" + self.update_end = False def get_self_ref(self): return self.base_var @@ -70,6 +71,8 @@ class RootMarshallingSource(MarshallingSource): if self.pointer: writer.assign(self.base_var, "(%s *)%s" % (self.c_type, self.pointer)) + if self.update_end: + writer.assign("end", "((uint8_t *)%s) + %s" % (self.base_var, self.sizeof)) else: writer.assign(self.base_var, "(%s *)end" % self.c_type) writer.increment("end", "%s" % self.sizeof) @@ -182,8 +185,18 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): element = "%s__element" % member.name + if not scope.variable_defined(element): + if array.ptr_array: + stars = " **" + else: + stars = " *" + scope.variable_def(element_type.c_type() + stars, element) + element_array = element + if array.ptr_array: + element = "*" + element + if not at_end: - writer.assign(element, container_src.get_ref(member.name)) + writer.assign(element_array, container_src.get_ref(member.name)) if is_byte_size: size_start_var = "%s__size_start" % member.name @@ -192,7 +205,6 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): with writer.index() as index: with writer.for_loop(index, nelements) as array_scope: - array_scope.variable_def(element_type.c_type() + " *", element) if at_end: writer.assign(element, "(%s *)end" % element_type.c_type()) writer.increment("end", element_type.sizeof()) @@ -201,13 +213,15 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): writer.statement("spice_marshaller_add_%s(m, *%s)" % (element_type.primitive_type(), element)) elif element_type.is_struct(): src2 = RootMarshallingSource(container_src, element_type.c_type(), element_type.sizeof(), element) + if array.is_extra_size(): + src2.update_end = True src2.reuse_scope = array_scope write_container_marshaller(writer, element_type, src2) else: writer.todo("array element unhandled type").newline() if not at_end: - writer.statement("%s++" % element) + writer.statement("%s++" % element_array) if is_byte_size: size_var = member.container.lookup_member(array.size[1]) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index b7bcac9..055034e 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -353,6 +353,7 @@ class ArrayType(Type): self.element_type = element_type self.size = size + self.ptr_array = False def __str__(self): if self.size == None: @@ -414,6 +415,9 @@ class ArrayType(Type): return [] raise Exception, "Pointer names in arrays not supported" + def is_extra_size(self): + return self.ptr_array + def contains_extra_size(self): return self.element_type.contains_extra_size() @@ -512,6 +516,8 @@ class Member(Containee): self.member_type.register() if self.has_attr("ptr32") and self.member_type.is_pointer(): self.member_type.set_ptr_size(4) + if self.has_attr("ptr_array") and self.member_type.is_array(): + self.member_type.ptr_array = True return self def is_primitive(self): @@ -523,7 +529,7 @@ class Member(Containee): return self.member_type.is_fixed_sizeof() def is_extra_size(self): - return self.has_end_attr() + return self.has_end_attr() or self.member_type.is_extra_size() def is_fixed_nw_size(self): if self.has_attr("virtual"): diff --git a/spice.proto b/spice.proto index 7528208..8df1238 100644 --- a/spice.proto +++ b/spice.proto @@ -564,12 +564,12 @@ struct String { string_flags flags; /* Special: Only one of a1/a4/a8 set */ switch (flags) { case RASTER_A1: - RasterGlyphA1 glyphs[length] @ctype(SpiceRasterGlyph); + RasterGlyphA1 glyphs[length] @ctype(SpiceRasterGlyph) @ptr_array; case RASTER_A4: - RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph); + RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph) @ptr_array; case RASTER_A8: - RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph); - } u @end @nomarshal; + RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph) @ptr_array; + } u @anon; }; channel DisplayChannel : BaseChannel { @@ -727,7 +727,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Text { - String *str; + String *str @marshall @nonnull; Rect back_area; Brush fore_brush @outvar(fore_brush); Brush back_brush @outvar(back_brush); diff --git a/spice1.proto b/spice1.proto index c9a18d9..982f666 100644 --- a/spice1.proto +++ b/spice1.proto @@ -522,12 +522,12 @@ struct String { string_flags flags; /* Special: Only one of a1/a4/a8 set */ switch (flags) { case RASTER_A1: - RasterGlyphA1 glyphs[length] @ctype(SpiceRasterGlyph); + RasterGlyphA1 glyphs[length] @ctype(SpiceRasterGlyph) @ptr_array; case RASTER_A4: - RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph); + RasterGlyphA4 glyphs[length] @ctype(SpiceRasterGlyph) @ptr_array; case RASTER_A8: - RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph); - } u @end @nomarshal; + RasterGlyphA8 glyphs[length] @ctype(SpiceRasterGlyph) @ptr_array; + } u @anon; }; channel DisplayChannel : BaseChannel { From fefc89c6c45a5371be611c83e570d9f3fbc7fe75 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 5 Jul 2010 12:03:34 +0200 Subject: [PATCH 046/147] marshaller: Add generic way to handle propagating attributes Also switches @ptr_array to use this --- python_modules/demarshal.py | 10 +++++----- python_modules/marshal.py | 4 ++-- python_modules/ptypes.py | 15 +++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 4d3e79b..5391e53 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -309,7 +309,7 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star if element_type.is_fixed_sizeof() and want_mem_size and not is_byte_size: # TODO: Overflow check the multiplication - if array.ptr_array: + if array.has_attr("ptr_array"): writer.assign(mem_size, "sizeof(void *) + SPICE_ALIGN(%s * %s, 4)" % (element_type.sizeof(), nelements)) else: writer.assign(mem_size, "%s * %s" % (element_type.sizeof(), nelements)) @@ -373,7 +373,7 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star if not array.is_extra_size(): writer.increment(mem_size, element_mem_size) if want_is_extra_size: - if array.ptr_array: + if array.has_attr("ptr_array"): writer.increment(extra_size, "sizeof(void *) + SPICE_ALIGN(%s, 4)" % element_mem_size) else: writer.increment(extra_size, "%s + %s" % (element_mem_size, element_extra_size)) @@ -741,7 +741,7 @@ def write_array_parser(writer, nelements, array, dest, scope): scope.variable_def("uint32_t", real_nelements) writer.assign("array_end", "end + %s" % nelements) writer.assign(real_nelements, 0) - if array.ptr_array: + if array.has_attr("ptr_array"): scope.variable_def("void **", "ptr_array") scope.variable_def("int", "ptr_array_index") writer.assign("ptr_array_index", 0) @@ -749,7 +749,7 @@ def write_array_parser(writer, nelements, array, dest, scope): writer.increment("end", "sizeof(void *) * %s" % nelements) with writer.index(no_block = is_byte_size) as index: with writer.while_loop("end < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: - if array.ptr_array: + if array.has_attr("ptr_array"): writer.statement("ptr_array[ptr_array_index++] = end") if is_byte_size: writer.increment(real_nelements, 1) @@ -760,7 +760,7 @@ def write_array_parser(writer, nelements, array, dest, scope): dest2 = dest.child_at_end(writer, element_type) dest2.reuse_scope = array_scope write_container_parser(writer, element_type, dest2) - if array.ptr_array: + if array.has_attr("ptr_array"): writer.comment("Align ptr_array element to 4 bytes").newline() writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") if is_byte_size: diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 95413fc..f151d94 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -186,13 +186,13 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): element = "%s__element" % member.name if not scope.variable_defined(element): - if array.ptr_array: + if array.has_attr("ptr_array"): stars = " **" else: stars = " *" scope.variable_def(element_type.c_type() + stars, element) element_array = element - if array.ptr_array: + if array.has_attr("ptr_array"): element = "*" + element if not at_end: diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 055034e..cc74b72 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -55,6 +55,13 @@ class FixedSize: s = s + " + ((minor >= %d)?%d:0)" % (i, self.vals[i]) return s +# Some attribute are propagated from member to the type as they really +# are part of the type definition, rather than the member. This applies +# only to attributes that affect pointer or array attributes, as these +# are member local types, unlike e.g. a Struct that may be used by +# other members +propagated_attributes=["ptr_array"] + class Type: def __init__(self): self.attributes = {} @@ -353,7 +360,6 @@ class ArrayType(Type): self.element_type = element_type self.size = size - self.ptr_array = False def __str__(self): if self.size == None: @@ -416,7 +422,7 @@ class ArrayType(Type): raise Exception, "Pointer names in arrays not supported" def is_extra_size(self): - return self.ptr_array + return self.has_attr("ptr_array") def contains_extra_size(self): return self.element_type.contains_extra_size() @@ -516,8 +522,9 @@ class Member(Containee): self.member_type.register() if self.has_attr("ptr32") and self.member_type.is_pointer(): self.member_type.set_ptr_size(4) - if self.has_attr("ptr_array") and self.member_type.is_array(): - self.member_type.ptr_array = True + for i in propagated_attributes: + if self.has_attr(i): + self.member_type.attributes[i] = self.attributes[i] return self def is_primitive(self): From f22381505d9ecc9fa6b5477d0a695b43957e93bb Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 5 Jul 2010 12:09:08 +0200 Subject: [PATCH 047/147] marshaller: Make @c_ptr a propagated attribute This simplifies some code --- python_modules/demarshal.py | 33 +++++++++++++++++---------------- python_modules/ptypes.py | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 5391e53..5a8f8ff 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -657,7 +657,7 @@ def write_switch_parser(writer, container, switch, dest, scope): if t.is_struct(): write_container_parser(writer, t, dest2) elif t.is_pointer(): - write_parse_pointer(writer, t, False, m.has_attr("c_ptr"), dest2, m.name, block) + write_parse_pointer(writer, t, False, dest2, m.name, block) elif t.is_primitive(): if m.has_attr("zero"): writer.statement("consume_%s(&in)" % (t.primitive_type())) @@ -766,21 +766,22 @@ def write_array_parser(writer, nelements, array, dest, scope): if is_byte_size: writer.assign(dest.get_ref(array.size[2]), real_nelements) -def write_parse_pointer(writer, t, at_end, as_c_ptr, dest, member_name, scope): - target_type = t.target_type - writer.assign("ptr_info[n_ptr].offset", "consume_%s(&in)" % t.primitive_type()) - writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) - if at_end: - writer.assign("ptr_info[n_ptr].dest", "end") - writer.increment("end", "sizeof(void *)" if as_c_ptr else "sizeof(SPICE_ADDRESS)"); - else: - writer.assign("ptr_info[n_ptr].dest", "&%s" % dest.get_ref(member_name)) - writer.assign("ptr_info[n_ptr].is_ptr", "1" if as_c_ptr else "0") - if target_type.is_array(): - nelements = read_array_len(writer, member_name, target_type, dest, scope) - writer.assign("ptr_info[n_ptr].nelements", nelements) +def write_parse_pointer(writer, t, at_end, dest, member_name, scope): + as_c_ptr = t.has_attr("c_ptr") + target_type = t.target_type + writer.assign("ptr_info[n_ptr].offset", "consume_%s(&in)" % t.primitive_type()) + writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) + if at_end: + writer.assign("ptr_info[n_ptr].dest", "end") + writer.increment("end", "sizeof(void *)" if as_c_ptr else "sizeof(SPICE_ADDRESS)"); + else: + writer.assign("ptr_info[n_ptr].dest", "&%s" % dest.get_ref(member_name)) + writer.assign("ptr_info[n_ptr].is_ptr", "1" if as_c_ptr else "0") + if target_type.is_array(): + nelements = read_array_len(writer, member_name, target_type, dest, scope) + writer.assign("ptr_info[n_ptr].nelements", nelements) - writer.statement("n_ptr++") + writer.statement("n_ptr++") def write_member_parser(writer, container, member, dest, scope): if member.has_attr("virtual"): @@ -798,7 +799,7 @@ def write_member_parser(writer, container, member, dest, scope): writer.comment("Reuse data from network message").newline() writer.assign(dest.get_ref(member.name), "(size_t)(message_start + consume_%s(&in))" % t.primitive_type()) else: - write_parse_pointer(writer, t, member.has_end_attr(), member.has_attr("c_ptr"), dest, member.name, scope) + write_parse_pointer(writer, t, member.has_end_attr(), dest, member.name, scope) elif t.is_primitive(): if member.has_attr("zero"): writer.statement("consume_%s(&in)" % t.primitive_type()) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index cc74b72..2c0dd88 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -60,7 +60,7 @@ class FixedSize: # only to attributes that affect pointer or array attributes, as these # are member local types, unlike e.g. a Struct that may be used by # other members -propagated_attributes=["ptr_array"] +propagated_attributes=["ptr_array", "c_ptr"] class Type: def __init__(self): From 32481bf381f97cf99bee6df36afadbb213b61e3f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 5 Jul 2010 12:13:45 +0200 Subject: [PATCH 048/147] marshaller: Make @nonnull a propagated attribute This cleans up some stuff --- python_modules/demarshal.py | 5 +---- python_modules/marshal.py | 2 +- python_modules/ptypes.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 5a8f8ff..5709567 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -82,7 +82,6 @@ class ItemInfo: self.prefix = prefix self.subprefix = prefix self.position = position - self.non_null = False self.member = None def nw_size(self): @@ -103,7 +102,6 @@ class MemberItemInfo(ItemInfo): self.type = member.member_type self.prefix = member.name self.subprefix = member.name - self.non_null = member.has_attr("nonnull") self.position = "(%s + %s)" % (start, container.get_nw_offset(member, "", "__nw_size")) self.member = member @@ -123,7 +121,6 @@ def write_validate_switch_member(writer, container, switch_member, scope, parent with writer.if_block(check, not first, False) as if_scope: item.type = c.member.member_type item.subprefix = item.prefix + "_" + m.name - item.non_null = c.member.has_attr("nonnull") sub_want_extra_size = want_extra_size if sub_want_extra_size and not m.contains_extra_size() and not m.is_extra_size(): writer.assign(item.extra_size(), 0) @@ -192,7 +189,7 @@ def write_validate_pointer_item(writer, container, item, scope, parent_scope, st target_type = item.type.target_type v = write_read_primitive_item(writer, item, scope) - if item.non_null: + if item.type.has_attr("nonnull"): writer.error_check("%s == 0" % v) # pointer target is struct, or array of primitives diff --git a/python_modules/marshal.py b/python_modules/marshal.py index f151d94..df0c3b3 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -235,7 +235,7 @@ def write_pointer_marshaller(writer, member, src): submarshaller = "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if member.get_fixed_nw_size() == 8 else 0) if member.has_attr("marshall"): writer.assign("m2", submarshaller) - if member.has_attr("nonnull"): + if t.has_attr("nonnull"): writer.statement("%s(m2, %s)" % (ptr_func, src.get_ref(member.name))) else: with writer.if_block("%s != NULL" % src.get_ref(member.name)) as block: diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 2c0dd88..31ae79d 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -60,7 +60,7 @@ class FixedSize: # only to attributes that affect pointer or array attributes, as these # are member local types, unlike e.g. a Struct that may be used by # other members -propagated_attributes=["ptr_array", "c_ptr"] +propagated_attributes=["ptr_array", "c_ptr", "nonnull"] class Type: def __init__(self): From daaf4865d81bb3d05e6fe799fd9f87d9950a1136 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 5 Jul 2010 13:13:09 +0200 Subject: [PATCH 049/147] marshaller: Add some docs describing the types of sizes --- python_modules/demarshal.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 5709567..cf6fefd 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1,6 +1,35 @@ import ptypes import codegen +# The handling of sizes is somewhat complex, as there are several types of size: +# * nw_size +# This is the network size, i.e. the number of bytes on the network +# +# * mem_size +# The total amount of memory used for the representation of something inside +# spice. This is generally sizeof(C struct), but can be larger if for instance +# the type has a variable size array at the end or has a pointer in it that +# points to another data chunk (which will be allocated after the main +# data chunk). This is essentially how much memory you need to allocate to +# contain the data type. +# +# * extra_size +# This is the size of anything that is not part of the containing structure. +# For instance, a primitive (say uint32_t) member has no extra size, because +# when allocating its part of the sizeof(MessageStructType) struct. However +# a variable array can be places at the end of a structure (@end) and its +# size is then extra_size. Note that this extra_size is included in the +# mem_size of the enclosing struct, and even if you request the mem_size +# of the array itself. However, extra_size is typically not requested +# when the full mem_size is also requested. +# +# extra sizes come in two flavours. contains_extra_size means that the item +# has a normal presence in the parent container, but has some additional +# extra_size it references. For instance via a pointer somewhere in it. +# There is also is_extra_size(). This indicates that the whole elements +# "normal" mem size should be considered extra size for the container, so +# when computing the parent mem_size you should add the mem_size of this +# part as extra_size def write_parser_helpers(writer): if writer.is_generated("helper", "demarshaller"): From bb1d862989ae4d69145e1c8b97dc83a6b8fb450c Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 5 Jul 2010 13:13:39 +0200 Subject: [PATCH 050/147] Handle extra size for switch and array the right way Even for is_extra_size() we should calculate the mem_size for arrays, its just that the parent type (in this case switch) should request mem_size if the type is_extra_size. --- python_modules/demarshal.py | 43 +++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index cf6fefd..606b926 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -150,13 +150,19 @@ def write_validate_switch_member(writer, container, switch_member, scope, parent with writer.if_block(check, not first, False) as if_scope: item.type = c.member.member_type item.subprefix = item.prefix + "_" + m.name - sub_want_extra_size = want_extra_size - if sub_want_extra_size and not m.contains_extra_size() and not m.is_extra_size(): - writer.assign(item.extra_size(), 0) - sub_want_extra_size = False + + all_as_extra_size = m.is_extra_size() and want_extra_size + if not want_mem_size and all_as_extra_size and not scope.variable_defined(item.mem_size()): + scope.variable_def("uint32_t", item.mem_size()) + + sub_want_mem_size = want_mem_size or all_as_extra_size + sub_want_extra_size = want_extra_size and not all_as_extra_size write_validate_item(writer, container, item, if_scope, scope, start, - want_nw_size, want_mem_size, sub_want_extra_size) + want_nw_size, sub_want_mem_size, sub_want_extra_size) + + if all_as_extra_size: + writer.assign(item.extra_size(), item.mem_size()) first = False @@ -341,7 +347,7 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star writer.assign(mem_size, "%s * %s" % (element_type.sizeof(), nelements)) want_mem_size = False - if not element_type.contains_extra_size() and not array.is_extra_size() and want_extra_size: + if not element_type.contains_extra_size() and want_extra_size: writer.assign(extra_size, 0) want_extra_size = False @@ -361,21 +367,14 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star element_extra_size = element_item.extra_size() scope.variable_def("uint32_t", element_nw_size) scope.variable_def("uint32_t", element_mem_size) - want_is_extra_size = False - want_element_mem_size = want_mem_size if want_extra_size: - if array.is_extra_size(): - want_is_extra_size = True - want_extra_size = False - want_element_mem_size = True - else: - scope.variable_def("uint32_t", element_extra_size) + scope.variable_def("uint32_t", element_extra_size) if want_nw_size: writer.assign(nw_size, 0) if want_mem_size: writer.assign(mem_size, 0) - if want_extra_size or want_is_extra_size: + if want_extra_size: writer.assign(extra_size, 0) want_element_nw_size = want_nw_size @@ -391,19 +390,16 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star with writer.index(no_block = is_byte_size) as index: with writer.while_loop("%s < %s" % (start2, start2_end) ) if is_byte_size else writer.for_loop(index, nelements) as scope: write_validate_item(writer, container, element_item, scope, parent_scope, start2, - want_element_nw_size, want_element_mem_size, want_extra_size) + want_element_nw_size, want_mem_size, want_extra_size) if want_nw_size: writer.increment(nw_size, element_nw_size) if want_mem_size: - if not array.is_extra_size(): - writer.increment(mem_size, element_mem_size) - if want_is_extra_size: if array.has_attr("ptr_array"): - writer.increment(extra_size, "sizeof(void *) + SPICE_ALIGN(%s, 4)" % element_mem_size) + writer.increment(mem_size, "sizeof(void *) + SPICE_ALIGN(%s, 4)" % element_mem_size) else: - writer.increment(extra_size, "%s + %s" % (element_mem_size, element_extra_size)) - elif want_extra_size: + writer.increment(mem_size, element_mem_size) + if want_extra_size: writer.increment(extra_size, element_extra_size) writer.increment(start2, start_increment) @@ -426,7 +422,8 @@ def write_validate_primitive_item(writer, container, item, scope, parent_scope, if want_mem_size: mem_size = item.mem_size() writer.assign(mem_size, item.type.sizeof()) - assert not want_extra_size + if want_extra_size: + writer.assign(item.extra_size(), 0) def write_validate_item(writer, container, item, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): From 6ca5b39e6f2a6d82d0234cc2967ebfce2b9092d6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 5 Jul 2010 20:45:13 +0200 Subject: [PATCH 051/147] Convert SpicePath.segments to a pointer array --- python_modules/demarshal.py | 51 ++++++++++++++++++++----------------- spice.proto | 2 +- spice1.proto | 4 +-- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 606b926..bd7660d 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -47,13 +47,16 @@ def write_parser_helpers(writer): swap = "SPICE_BYTESWAP%d" % size if size == 8: writer.macro("read_%s" % type, "ptr", "(*((%s_t *)(ptr)))" % type) + writer.macro("write_%s" % type, "ptr, val", "*(%s_t *)(ptr) = val" % (type)) else: writer.macro("read_%s" % type, "ptr", "((%s_t)%s(*((%s_t *)(ptr)))" % (type, swap, utype)) + writer.macro("write_%s" % type, "ptr, val", "*(%s_t *)(ptr) = %s((%s_t)val)" % (utype, swap, utype)) writer.writeln("#else") for size in [8, 16, 32, 64]: for sign in ["", "u"]: type = "%sint%d" % (sign, size) writer.macro("read_%s" % type, "ptr", "(*((%s_t *)(ptr)))" % type) + writer.macro("write_%s" % type, "ptr, val", "(*((%s_t *)(ptr))) = val" % type) writer.writeln("#endif") for size in [8, 16, 32, 64]: @@ -96,6 +99,15 @@ def write_read_primitive(writer, start, container, name, scope): writer.assign(var, "read_%s(pos)" % (m.member_type.primitive_type())) return var +def write_write_primitive(writer, start, container, name, val): + m = container.lookup_member(name) + assert(m.is_primitive()) + writer.assign("pos", start + " + " + container.get_nw_offset(m, "", "__nw_size")) + + var = "%s__value" % (name) + writer.statement("write_%s(pos, %s)" % (m.member_type.primitive_type(), val)) + return var + def write_read_primitive_item(writer, item, scope): assert(item.type.is_primitive()) writer.assign("pos", item.get_position()) @@ -280,6 +292,9 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star element_type = array.element_type if array.is_bytes_length(): nelements = "%s__nbytes" %(item.prefix) + real_nelements = "%s__nelements" %(item.prefix) + if not parent_scope.variable_defined(real_nelements): + parent_scope.variable_def("uint32_t", real_nelements) else: nelements = "%s__nelements" %(item.prefix) if not parent_scope.variable_defined(nelements): @@ -315,6 +330,7 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star is_byte_size = True v = write_read_primitive(writer, start, container, array.size[1], scope) writer.assign(nelements, v) + writer.assign(real_nelements, 0) elif array.is_cstring_length(): writer.todo("cstring array size type not handled yet") else: @@ -389,6 +405,8 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star with writer.index(no_block = is_byte_size) as index: with writer.while_loop("%s < %s" % (start2, start2_end) ) if is_byte_size else writer.for_loop(index, nelements) as scope: + if is_byte_size: + writer.increment(real_nelements, 1) write_validate_item(writer, container, element_item, scope, parent_scope, start2, want_element_nw_size, want_mem_size, want_extra_size) @@ -405,6 +423,7 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star writer.increment(start2, start_increment) if is_byte_size: writer.error_check("%s != %s" % (start2, start2_end)) + write_write_primitive(writer, start, container, array.size[1], real_nelements) def write_validate_struct_item(writer, container, item, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): @@ -613,11 +632,9 @@ class SubDemarshallingDestination(DemarshallingDestination): def get_ref(self, member): return self.parent_dest.get_ref(self.member) + "." + member -def read_array_len(writer, prefix, array, dest, scope, handles_bytes = False): - if array.is_bytes_length(): - nelements = "%s__nbytes" % prefix - else: - nelements = "%s__nelements" % prefix +# Note: during parsing, byte_size types have been converted to count during validation +def read_array_len(writer, prefix, array, dest, scope): + nelements = "%s__nelements" % prefix if dest.is_toplevel(): return nelements # Already there for toplevel, need not recalculate element_type = array.element_type @@ -645,9 +662,7 @@ def read_array_len(writer, prefix, array, dest, scope, handles_bytes = False): else: writer.assign(nelements, "((%s * %s + 7) / 8 ) * %s" % (bpp, width_v, rows_v)) elif array.is_bytes_length(): - if not handles_bytes: - raise NotImplementedError("handling of bytes() not supported here yet") - writer.assign(nelements, array.size[1]) + writer.assign(nelements, dest.get_ref(array.size[2])) else: raise NotImplementedError("TODO array size type not handled yet") return nelements @@ -758,24 +773,16 @@ def write_array_parser(writer, nelements, array, dest, scope): writer.increment("in", nelements) writer.increment("end", nelements) else: - if is_byte_size: - real_nelements = nelements[:-len("nbytes")] + "nelements" - scope.variable_def("uint8_t *", "array_end") - scope.variable_def("uint32_t", real_nelements) - writer.assign("array_end", "end + %s" % nelements) - writer.assign(real_nelements, 0) if array.has_attr("ptr_array"): scope.variable_def("void **", "ptr_array") scope.variable_def("int", "ptr_array_index") writer.assign("ptr_array_index", 0) writer.assign("ptr_array", "(void **)end") writer.increment("end", "sizeof(void *) * %s" % nelements) - with writer.index(no_block = is_byte_size) as index: - with writer.while_loop("end < array_end") if is_byte_size else writer.for_loop(index, nelements) as array_scope: + with writer.index() as index: + with writer.for_loop(index, nelements) as array_scope: if array.has_attr("ptr_array"): writer.statement("ptr_array[ptr_array_index++] = end") - if is_byte_size: - writer.increment(real_nelements, 1) if element_type.is_primitive(): writer.statement("*(%s *)end = consume_%s(&in)" % (element_type.c_type(), element_type.primitive_type())) writer.increment("end", element_type.sizeof()) @@ -786,8 +793,6 @@ def write_array_parser(writer, nelements, array, dest, scope): if array.has_attr("ptr_array"): writer.comment("Align ptr_array element to 4 bytes").newline() writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") - if is_byte_size: - writer.assign(dest.get_ref(array.size[2]), real_nelements) def write_parse_pointer(writer, t, at_end, dest, member_name, scope): as_c_ptr = t.has_attr("c_ptr") @@ -831,14 +836,14 @@ def write_member_parser(writer, container, member, dest, scope): writer.increment("end", t.sizeof()) else: if member.has_attr("bytes_count"): - scope.variable_def("uint32_t", member.name); - dest_var = member.name + print member.attributes["bytes_count"] + dest_var = dest.get_ref(member.attributes["bytes_count"][0]) else: dest_var = dest.get_ref(member.name) writer.assign(dest_var, "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums elif t.is_array(): - nelements = read_array_len(writer, member.name, t, dest, scope, handles_bytes = True) + nelements = read_array_len(writer, member.name, t, dest, scope) if member.has_attr("as_ptr") and t.element_type.is_fixed_nw_size(): writer.comment("use array as pointer").newline() writer.assign(dest.get_ref(member.name), "(%s *)in" % t.element_type.c_type()) diff --git a/spice.proto b/spice.proto index 8df1238..e437630 100644 --- a/spice.proto +++ b/spice.proto @@ -393,7 +393,7 @@ struct PathSegment { struct Path { uint32 num_segments; - PathSegment segments[num_segments] @end; + PathSegment segments[num_segments] @ptr_array; }; struct Clip { diff --git a/spice1.proto b/spice1.proto index 982f666..b49371a 100644 --- a/spice1.proto +++ b/spice1.proto @@ -374,8 +374,8 @@ struct PathSegment { } @ctype(SpicePathSeg); struct Path { - uint32 segments_size @bytes_count; - PathSegment segments[bytes(segments_size, num_segments)] @end; + uint32 segments_size @bytes_count(num_segments); + PathSegment segments[bytes(segments_size, num_segments)] @ptr_array; }; struct Clip { From 5492b3ccae766cc794df142ec493eb15be48f04a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 6 Jul 2010 22:12:26 +0200 Subject: [PATCH 052/147] marshaller: Make get_nw_offset() handle deep member references --- python_modules/ptypes.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 31ae79d..f02437a 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -92,6 +92,9 @@ class Type: def is_array(self): return isinstance(self, ArrayType) + def contains_member(self, member): + return False + def is_struct(self): return isinstance(self, StructType) @@ -527,6 +530,9 @@ class Member(Containee): self.member_type.attributes[i] = self.attributes[i] return self + def contains_member(self, member): + return self.member_type.contains_member(member) + def is_primitive(self): return self.member_type.is_primitive() @@ -694,6 +700,9 @@ class Switch(Containee): return "sizeof(((%s *)NULL)->%s)" % (self.container.c_type(), self.name) + def contains_member(self, member): + return False # TODO: Don't support switch deep member lookup yet + def has_pointer(self): for c in self.cases: if c.has_pointer(): @@ -739,11 +748,20 @@ class ContainerType(Type): size = size + i.get_fixed_nw_size() return size + def contains_member(self, member): + for m in self.members: + if m == member or m.contains_member(member): + return True + return False + def get_fixed_nw_offset(self, member): size = 0 for i in self.members: if i == member: break + if i.contains_member(member): + size = size + i.member_type.get_fixed_nw_offset(member) + break if i.is_fixed_nw_size(): size = size + i.get_fixed_nw_size() return size @@ -773,13 +791,20 @@ class ContainerType(Type): def get_nw_offset(self, member, prefix = "", postfix = ""): fixed = self.get_fixed_nw_offset(member) v = [] - for m in self.members: - if m == member: - break - if m.is_switch() and m.has_switch_member(member): - break - if not m.is_fixed_nw_size(): - v.append(prefix + m.name + postfix) + container = self + while container != None: + members = container.members + container = None + for m in members: + if m == member: + break + if m.contains_member(member): + container = m.member_type + break + if m.is_switch() and m.has_switch_member(member): + break + if not m.is_fixed_nw_size(): + v.append(prefix + m.name + postfix) if len(v) > 0: return str(fixed) + " + " + (" + ".join(v)) else: From 1b432de3b96c80e05287b6785b5be624baae8ba3 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 6 Jul 2010 22:13:18 +0200 Subject: [PATCH 053/147] Add support for @chunk --- python_modules/demarshal.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index bd7660d..51dbe31 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -261,7 +261,9 @@ def write_validate_pointer_item(writer, container, item, scope, parent_scope, st writer.error_check("message_start + %s + %s > message_end" % (v, array_item.nw_size())) if want_extra_size: - if item.member and item.member.has_attr("nocopy"): + if item.member and item.member.has_attr("chunk"): + writer.assign(item.extra_size(), "sizeof(SpiceChunks) + sizeof(SpiceChunk)") + elif item.member and item.member.has_attr("nocopy"): writer.comment("@nocopy, so no extra size").newline() writer.assign(item.extra_size(), 0) elif target_type.element_type.get_fixed_nw_size == 1: @@ -823,7 +825,20 @@ def write_member_parser(writer, container, member, dest, scope): t = member.member_type if t.is_pointer(): - if member.has_attr("nocopy"): + if member.has_attr("chunk"): + assert(t.target_type.is_array()) + nelements = read_array_len(writer, member.name, t.target_type, dest, scope) + writer.comment("Reuse data from network message as chunk").newline() + scope.variable_def("SpiceChunks *", "chunks"); + writer.assign("chunks", "(SpiceChunks *)end") + writer.increment("end", "sizeof(SpiceChunks) + sizeof(SpiceChunk)") + writer.assign(dest.get_ref(member.name), "chunks") # spice_chunks_new_linear(message_start + consume_%s(&in), %s)" % (t.primitive_type(), nelements)) + writer.assign("chunks->data_size", nelements) + writer.assign("chunks->flags", 0) + writer.assign("chunks->num_chunks", 1) + writer.assign("chunks->chunk[0].len", nelements) + writer.assign("chunks->chunk[0].data", "message_start + consume_%s(&in)" % t.primitive_type()) + elif member.has_attr("nocopy"): writer.comment("Reuse data from network message").newline() writer.assign(dest.get_ref(member.name), "(size_t)(message_start + consume_%s(&in))" % t.primitive_type()) else: @@ -1140,6 +1155,7 @@ def write_includes(writer): writer.writeln("#include ") writer.writeln("#include ") writer.writeln("#include ") + writer.writeln("#include ") writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") From 4a12b9c0dd555695177a017f4826557dde5fbe11 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 7 Jul 2010 20:40:06 +0200 Subject: [PATCH 054/147] codegen: support @chunk on non-pointer arrays This is similar to @as_ptr, but generates a single chunk of data. --- python_modules/demarshal.py | 30 +++++++++++++++++++++++++++--- python_modules/ptypes.py | 4 ++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 51dbe31..323fc70 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -357,6 +357,18 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star writer.assign(nw_size, "(%s) * %s" % (element_size, nelements)) want_nw_size = False + if array.has_attr("as_ptr") and want_mem_size: + writer.assign(mem_size, "sizeof(void *)") + want_mem_size = False + + if array.has_attr("chunk"): + if want_mem_size: + writer.assign(extra_size, "sizeof(SpiceChunks *)") + want_mem_size = False + if want_extra_size: + writer.assign(extra_size, "sizeof(SpiceChunks) + sizeof(SpiceChunk)") + want_extra_size = False + if element_type.is_fixed_sizeof() and want_mem_size and not is_byte_size: # TODO: Overflow check the multiplication if array.has_attr("ptr_array"): @@ -832,7 +844,7 @@ def write_member_parser(writer, container, member, dest, scope): scope.variable_def("SpiceChunks *", "chunks"); writer.assign("chunks", "(SpiceChunks *)end") writer.increment("end", "sizeof(SpiceChunks) + sizeof(SpiceChunk)") - writer.assign(dest.get_ref(member.name), "chunks") # spice_chunks_new_linear(message_start + consume_%s(&in), %s)" % (t.primitive_type(), nelements)) + writer.assign(dest.get_ref(member.name), "chunks") writer.assign("chunks->data_size", nelements) writer.assign("chunks->flags", 0) writer.assign("chunks->num_chunks", 1) @@ -851,7 +863,6 @@ def write_member_parser(writer, container, member, dest, scope): writer.increment("end", t.sizeof()) else: if member.has_attr("bytes_count"): - print member.attributes["bytes_count"] dest_var = dest.get_ref(member.attributes["bytes_count"][0]) else: dest_var = dest.get_ref(member.name) @@ -859,7 +870,20 @@ def write_member_parser(writer, container, member, dest, scope): #TODO validate e.g. flags and enums elif t.is_array(): nelements = read_array_len(writer, member.name, t, dest, scope) - if member.has_attr("as_ptr") and t.element_type.is_fixed_nw_size(): + if member.has_attr("chunk") and t.element_type.is_fixed_nw_size() and t.element_type.get_fixed_nw_size() == 1: + writer.comment("use array as chunk").newline() + + scope.variable_def("SpiceChunks *", "chunks"); + writer.assign("chunks", "(SpiceChunks *)end") + writer.increment("end", "sizeof(SpiceChunks) + sizeof(SpiceChunk)") + writer.assign(dest.get_ref(member.name), "chunks") + writer.assign("chunks->data_size", nelements) + writer.assign("chunks->flags", 0) + writer.assign("chunks->num_chunks", 1) + writer.assign("chunks->chunk[0].len", nelements) + writer.assign("chunks->chunk[0].data", "in") + writer.increment("in", "%s" % (nelements)) + elif member.has_attr("as_ptr") and t.element_type.is_fixed_nw_size(): writer.comment("use array as pointer").newline() writer.assign(dest.get_ref(member.name), "(%s *)in" % t.element_type.c_type()) len_var = member.attributes["as_ptr"] diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index f02437a..68cf3df 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -60,7 +60,7 @@ class FixedSize: # only to attributes that affect pointer or array attributes, as these # are member local types, unlike e.g. a Struct that may be used by # other members -propagated_attributes=["ptr_array", "c_ptr", "nonnull"] +propagated_attributes=["ptr_array", "c_ptr", "nonnull", "chunk"] class Type: def __init__(self): @@ -428,7 +428,7 @@ class ArrayType(Type): return self.has_attr("ptr_array") def contains_extra_size(self): - return self.element_type.contains_extra_size() + return self.element_type.contains_extra_size() or self.has_attr("chunk") def sizeof(self): return "%s * %s" % (self.element_type.sizeof(), self.size) From 0ed056da99f2ee23da5600f76d5824c68918c1b5 Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Thu, 1 Jul 2010 17:55:33 +0200 Subject: [PATCH 055/147] Properly parse QXLImage to the new-world SpiceImage SpiceImage now replaces RedImage and has all image types in it. All image data are now chunked (and as such not copied when demarshalling). --- python_modules/demarshal.py | 4 +- python_modules/ptypes.py | 31 ++++++++++--- python_modules/spice_parser.py | 2 +- spice.proto | 81 ++++++++++++++++++---------------- spice1.proto | 67 ++++++++++++++-------------- 5 files changed, 105 insertions(+), 80 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 323fc70..1ae1911 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -93,7 +93,7 @@ def write_read_primitive(writer, start, container, name, scope): writer.assign("pos", start + " + " + container.get_nw_offset(m, "", "__nw_size")) writer.error_check("pos + %s > message_end" % m.member_type.get_fixed_nw_size()) - var = "%s__value" % (name) + var = "%s__value" % (name.replace(".", "_")) if not scope.variable_defined(var): scope.variable_def(m.member_type.c_type(), var) writer.assign(var, "read_%s(pos)" % (m.member_type.primitive_type())) @@ -112,7 +112,7 @@ def write_read_primitive_item(writer, item, scope): assert(item.type.is_primitive()) writer.assign("pos", item.get_position()) writer.error_check("pos + %s > message_end" % item.type.get_fixed_nw_size()) - var = "%s__value" % (item.subprefix) + var = "%s__value" % (item.subprefix.replace(".", "_")) scope.variable_def(item.type.c_type(), var) writer.assign(var, "read_%s(pos)" % (item.type.primitive_type())) return var diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 68cf3df..715544f 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -811,14 +811,31 @@ class ContainerType(Type): return str(fixed) def lookup_member(self, name): + dot = name.find('.') + rest = None + if dot >= 0: + rest = name[dot+1:] + name = name[:dot] + + member = None if self.members_by_name.has_key(name): - return self.members_by_name[name] - for m in self.members: - if m.is_switch(): - member = m.lookup_case_member(name) - if member: - return member - raise Exception, "No member called %s found" % name + member = self.members_by_name[name] + else: + for m in self.members: + if m.is_switch(): + member = m.lookup_case_member(name) + if member != None: + break + if member != None: + break + + if member == None: + raise Exception, "No member called %s found" % name + + if rest != None: + return member.member_type.lookup_member(rest) + + return member class StructType(ContainerType): def __init__(self, name, members, attribute_list): diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index 61ef458..ac2da8d 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -95,7 +95,7 @@ def SPICE_BNF(): switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | case_.suppress() + identifier + colon)) + variableDef) \ .setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) - switchBody = Group(switch_ + lparen + identifier + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \ + switchBody = Group(switch_ + lparen + delimitedList(identifier,delim='.', combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \ .setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace) structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction(lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3])) diff --git a/spice.proto b/spice.proto index e437630..e16250f 100644 --- a/spice.proto +++ b/spice.proto @@ -111,7 +111,7 @@ channel BaseChannel { message { uint32 id; uint64 timestamp; - uint8 data[] @end @ctype(uint8_t) @as_ptr(data_len); + uint8 data[] @ctype(uint8_t) @as_ptr(data_len); } ping; message { @@ -435,16 +435,16 @@ struct BitmapData { uint32 stride; switch (flags) { case PAL_FROM_CACHE: - uint64 palette; + uint64 palette_id; default: - Palette *palette @outvar(bitmap); + Palette *palette @outvar(bitmap) @c_ptr; } pal @anon; - uint8 *data[image_size(8, stride, y)] @nocopy; /* pointer to array, not array of pointers as in C */ + uint8 *data[image_size(8, stride, y)] @chunk; /* pointer to array, not array of pointers as in C */ } @ctype(SpiceBitmap); struct BinaryData { uint32 data_size; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @nomarshal @chunk; } @ctype(SpiceQUICData); struct LZPLTData { @@ -452,58 +452,63 @@ struct LZPLTData { uint32 data_size; switch (flags) { case PAL_FROM_CACHE: - uint64 palette; + uint64 palette_id; default: - Palette *palette @nonnull @outvar(lzplt); + Palette *palette @nonnull @outvar(lzplt) @c_ptr; } pal @anon; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @nomarshal @chunk; }; struct ZlibGlzRGBData { uint32 glz_data_size; uint32 data_size; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @nomarshal @chunk; } @ctype(SpiceZlibGlzRGBData); struct JPEGAlphaData { jpeg_alpha_flags flags; uint32 jpeg_size; uint32 data_size; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @nomarshal @chunk; } @ctype(SpiceJPEGAlphaData); struct Surface { uint32 surface_id; }; -struct Image { - uint64 id; - image_type type; - image_flags flags; - uint32 width; - uint32 height; - switch (type) { +struct Image { + struct ImageDescriptor { + uint64 id; + image_type type; + image_flags flags; + uint32 width; + uint32 height; + } descriptor; + + switch (descriptor.type) { case BITMAP: - BitmapData bitmap_data @ctype(SpiceBitmap); + BitmapData bitmap; case QUIC: + BinaryData quic; case LZ_RGB: case GLZ_RGB: + BinaryData lz_rgb; case JPEG: - BinaryData binary_data @ctype(SpiceQUICData); + BinaryData jpeg; case LZ_PLT: - LZPLTData lzplt_data @ctype(SpiceLZPLTData); + LZPLTData lz_plt; case ZLIB_GLZ_RGB: - ZlibGlzRGBData zlib_glz_data @ctype(SpiceZlibGlzRGBData); + ZlibGlzRGBData zlib_glz; case JPEG_ALPHA: - JPEGAlphaData jpeg_alpha_data @ctype(SpiceJPEGAlphaData); + JPEGAlphaData jpeg_alpha; case SURFACE: - Surface surface_data; - } u @end; -} @ctype(SpiceImageDescriptor); + Surface surface; + } u; +}; struct Pattern { - Image *pat @nonnull; + Image *pat @nonnull @c_ptr; Point pos; }; @@ -520,7 +525,7 @@ struct Brush { struct QMask { mask_flags flags; Point pos; - Image *bitmap; + Image *bitmap @c_ptr; }; struct LineAttr { @@ -649,7 +654,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Opaque { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; Brush brush; ropd rop_descriptor; @@ -661,7 +666,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Copy { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -672,7 +677,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Blend { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -704,7 +709,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Rop3 { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; Brush brush; uint8 rop3; @@ -739,7 +744,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Transparent { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; uint32 src_color; uint32 true_color; @@ -751,7 +756,7 @@ channel DisplayChannel : BaseChannel { struct AlphaBlnd { alpha_flags alpha_flags; uint8 alpha; - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; } data; } draw_alpha_blend; @@ -875,7 +880,7 @@ struct CursorHeader { struct Cursor { cursor_flags flags; CursorHeader header; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); }; channel CursorChannel : BaseChannel { @@ -929,13 +934,13 @@ channel PlaybackChannel : BaseChannel { server: message { uint32 time; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); } @ctype(SpiceMsgPlaybackPacket) data = 101; message { uint32 time; audio_data_mode mode; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); } mode; message { @@ -960,13 +965,13 @@ channel RecordChannel : BaseChannel { client: message { uint32 time; - uint8 data[] @end @nomarshal @as_ptr(data_size); + uint8 data[] @nomarshal @as_ptr(data_size); } @ctype(SpiceMsgcRecordPacket) data = 101; message { uint32 time; audio_data_mode mode; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); } mode; message { diff --git a/spice1.proto b/spice1.proto index b49371a..7bbccb7 100644 --- a/spice1.proto +++ b/spice1.proto @@ -111,7 +111,7 @@ channel BaseChannel { message { uint32 id; uint64 timestamp; - uint8 data[] @end @ctype(uint8_t) @as_ptr(data_len); + uint8 data[] @ctype(uint8_t) @as_ptr(data_len); } ping; message { @@ -419,16 +419,16 @@ struct BitmapData { uint32 stride; switch (flags) { case PAL_FROM_CACHE: - uint64 palette; + uint64 palette_id; default: - Palette *palette @outvar(bitmap); + Palette *palette @outvar(bitmap) @c_ptr; } pal @anon; - uint8 *data[image_size(8, stride, y)] @nocopy; /* pointer to array, not array of pointers as in C */ + uint8 *data[image_size(8, stride, y)] @chunk; /* pointer to array, not array of pointers as in C */ } @ctype(SpiceBitmap); struct BinaryData { uint32 data_size; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @nomarshal @chunk; } @ctype(SpiceQUICData); struct LZPLTData { @@ -436,34 +436,37 @@ struct LZPLTData { uint32 data_size; switch (flags) { case PAL_FROM_CACHE: - uint64 palette; + uint64 palette_id; default: - Palette *palette @nonnull @outvar(lzplt); + Palette *palette @nonnull @outvar(lzplt) @c_ptr; } pal @anon; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @nomarshal @chunk; }; struct Image { - uint64 id; - image_type type; - image_flags flags; - uint32 width; - uint32 height; + struct ImageDescriptor { + uint64 id; + image_type type; + image_flags flags; + uint32 width; + uint32 height; + } descriptor; - switch (type) { + switch (descriptor.type) { case BITMAP: - BitmapData bitmap_data @ctype(SpiceBitmap); + BitmapData bitmap; case QUIC: + BinaryData quic; case LZ_RGB: case GLZ_RGB: - BinaryData binary_data @ctype(SpiceQUICData); + BinaryData lz_rgb; case LZ_PLT: - LZPLTData lzplt_data @ctype(SpiceLZPLTData); - } u @end; -} @ctype(SpiceImageDescriptor); + LZPLTData lz_plt; + } u; +}; struct Pattern { - Image *pat @nonnull; + Image *pat @nonnull @c_ptr; Point pos; }; @@ -480,7 +483,7 @@ struct Brush { struct QMask { mask_flags flags; Point pos; - Image *bitmap; + Image *bitmap @c_ptr; }; struct LineAttr { @@ -608,7 +611,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Opaque { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; Brush brush; ropd rop_descriptor; @@ -620,7 +623,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Copy { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -631,7 +634,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Blend { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -663,7 +666,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Rop3 { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; Brush brush; uint8 rop3; @@ -698,7 +701,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Transparent { - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; uint32 src_color; uint32 true_color; @@ -710,7 +713,7 @@ channel DisplayChannel : BaseChannel { struct AlphaBlnd { int8 alpha_flags @virtual(0); uint8 alpha; - Image *src_bitmap; + Image *src_bitmap @c_ptr; Rect src_area; } data; } draw_alpha_blend; @@ -822,7 +825,7 @@ struct CursorHeader { struct Cursor { cursor_flags flags; CursorHeader header; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); }; channel CursorChannel : BaseChannel { @@ -876,13 +879,13 @@ channel PlaybackChannel : BaseChannel { server: message { uint32 time; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); } @ctype(SpiceMsgPlaybackPacket) data = 101; message { uint32 time; audio_data_mode mode; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); } mode; message { @@ -907,13 +910,13 @@ channel RecordChannel : BaseChannel { client: message { uint32 time; - uint8 data[] @end @nomarshal @as_ptr(data_size); + uint8 data[] @nomarshal @as_ptr(data_size); } @ctype(SpiceMsgcRecordPacket) data = 101; message { uint32 time; audio_data_mode mode; - uint8 data[] @end @as_ptr(data_size); + uint8 data[] @as_ptr(data_size); } mode; message { From 2952d5bd9a7bc21cff4d4035b597682339dc02cf Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 8 Jul 2010 12:43:33 +0200 Subject: [PATCH 056/147] Properly parse QXLLineAttrs.style --- spice.proto | 2 +- spice1.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index e16250f..bd56eb2 100644 --- a/spice.proto +++ b/spice.proto @@ -536,7 +536,7 @@ struct LineAttr { } u1 @anon; switch (flags) { case STYLED: - fixed28_4 *style[style_nseg]; + fixed28_4 *style[style_nseg] @c_ptr; } u2 @anon; }; diff --git a/spice1.proto b/spice1.proto index 7bbccb7..23ce9f1 100644 --- a/spice1.proto +++ b/spice1.proto @@ -493,7 +493,7 @@ struct LineAttr { uint8 style_nseg; fixed28_4 width @zero; fixed28_4 miter_limit @zero; - fixed28_4 *style[style_nseg]; + fixed28_4 *style[style_nseg] @c_ptr; }; struct RasterGlyphA1 { From 74f9cd9f731f33c8c435fafcf3bada7ef0661470 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 8 Jul 2010 13:07:17 +0200 Subject: [PATCH 057/147] codegen: No SPICE_ADDRESS types left, drop @c_ptr --- python_modules/demarshal.py | 24 +++++++----------------- python_modules/ptypes.py | 7 +++++-- spice.proto | 26 +++++++++++++------------- spice1.proto | 24 ++++++++++++------------ 4 files changed, 37 insertions(+), 44 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 1ae1911..e8afabb 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -82,9 +82,8 @@ def write_parser_helpers(writer): writer.begin_block("struct PointerInfo") writer.variable_def("uint64_t", "offset") writer.variable_def("parse_func_t", "parse") - writer.variable_def("void *", "dest") + writer.variable_def("void **", "dest") writer.variable_def("uint32_t", "nelements") - writer.variable_def("int", "is_ptr") writer.end_block(semicolon=True) def write_read_primitive(writer, start, container, name, scope): @@ -195,7 +194,7 @@ def write_validate_struct_function(writer, struct): writer.set_is_generated("validator", validate_function) writer = writer.function_helper() - scope = writer.function(validate_function, "static intptr_t", "uint8_t *message_start, uint8_t *message_end, SPICE_ADDRESS offset, int minor") + scope = writer.function(validate_function, "static intptr_t", "uint8_t *message_start, uint8_t *message_end, uint64_t offset, int minor") scope.variable_def("uint8_t *", "start = message_start + offset") scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); scope.variable_def("size_t", "mem_size", "nw_size"); @@ -809,16 +808,14 @@ def write_array_parser(writer, nelements, array, dest, scope): writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") def write_parse_pointer(writer, t, at_end, dest, member_name, scope): - as_c_ptr = t.has_attr("c_ptr") target_type = t.target_type writer.assign("ptr_info[n_ptr].offset", "consume_%s(&in)" % t.primitive_type()) writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) if at_end: - writer.assign("ptr_info[n_ptr].dest", "end") - writer.increment("end", "sizeof(void *)" if as_c_ptr else "sizeof(SPICE_ADDRESS)"); + writer.assign("ptr_info[n_ptr].dest", "(void **)end") + writer.increment("end", "sizeof(void *)"); else: - writer.assign("ptr_info[n_ptr].dest", "&%s" % dest.get_ref(member_name)) - writer.assign("ptr_info[n_ptr].is_ptr", "1" if as_c_ptr else "0") + writer.assign("ptr_info[n_ptr].dest", "(void **)&%s" % dest.get_ref(member_name)) if target_type.is_array(): nelements = read_array_len(writer, member_name, target_type, dest, scope) writer.assign("ptr_info[n_ptr].nelements", nelements) @@ -932,20 +929,13 @@ def write_ptr_info_check(writer): with writer.for_loop(index, "n_ptr") as scope: offset = "ptr_info[%s].offset" % index function = "ptr_info[%s].parse" % index - is_ptr = "ptr_info[%s].is_ptr" % index dest = "ptr_info[%s].dest" % index with writer.if_block("%s == 0" % offset, newline=False): - with writer.if_block(is_ptr, newline=False): - writer.assign("*(void **)(%s)" % dest, "NULL") - with writer.block(" else"): - writer.assign("*(SPICE_ADDRESS *)(%s)" % dest, "0") + writer.assign("*%s" % dest, "NULL") with writer.block(" else"): writer.comment("Align to 32 bit").newline() writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") - with writer.if_block(is_ptr, newline=False): - writer.assign("*(void **)(%s)" % dest, "(void *)end") - with writer.block(" else"): - writer.assign("*(SPICE_ADDRESS *)(%s)" % dest, "(SPICE_ADDRESS)end") + writer.assign("*%s" % dest, "(void *)end") writer.assign("end", "%s(message_start, message_end, end, &ptr_info[%s], minor)" % (function, index)) writer.error_check("end == NULL") writer.newline() diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 715544f..5e18aa3 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -60,7 +60,7 @@ class FixedSize: # only to attributes that affect pointer or array attributes, as these # are member local types, unlike e.g. a Struct that may be used by # other members -propagated_attributes=["ptr_array", "c_ptr", "nonnull", "chunk"] +propagated_attributes=["ptr_array", "nonnull", "chunk"] class Type: def __init__(self): @@ -469,7 +469,10 @@ class PointerType(Type): return self.pointer_size def c_type(self): - return "SPICE_ADDRESS" + if self.pointer_size == 4: + return "uint32_t" + else: + return "uint64_t" def has_pointer(self): return True diff --git a/spice.proto b/spice.proto index bd56eb2..f6aa34e 100644 --- a/spice.proto +++ b/spice.proto @@ -400,7 +400,7 @@ struct Clip { clip_type type; switch (type) { case RECTS: - ClipRects *rects @outvar(cliprects) @c_ptr @marshall @nonnull; + ClipRects *rects @outvar(cliprects) @marshall @nonnull; } u @anon; }; @@ -437,7 +437,7 @@ struct BitmapData { case PAL_FROM_CACHE: uint64 palette_id; default: - Palette *palette @outvar(bitmap) @c_ptr; + Palette *palette @outvar(bitmap); } pal @anon; uint8 *data[image_size(8, stride, y)] @chunk; /* pointer to array, not array of pointers as in C */ } @ctype(SpiceBitmap); @@ -454,7 +454,7 @@ struct LZPLTData { case PAL_FROM_CACHE: uint64 palette_id; default: - Palette *palette @nonnull @outvar(lzplt) @c_ptr; + Palette *palette @nonnull @outvar(lzplt); } pal @anon; uint8 data[data_size] @nomarshal @chunk; }; @@ -508,7 +508,7 @@ struct Image { }; struct Pattern { - Image *pat @nonnull @c_ptr; + Image *pat @nonnull; Point pos; }; @@ -525,7 +525,7 @@ struct Brush { struct QMask { mask_flags flags; Point pos; - Image *bitmap @c_ptr; + Image *bitmap; }; struct LineAttr { @@ -536,7 +536,7 @@ struct LineAttr { } u1 @anon; switch (flags) { case STYLED: - fixed28_4 *style[style_nseg] @c_ptr; + fixed28_4 *style[style_nseg]; } u2 @anon; }; @@ -654,7 +654,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Opaque { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; Brush brush; ropd rop_descriptor; @@ -666,7 +666,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Copy { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -677,7 +677,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Blend { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -709,7 +709,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Rop3 { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; Brush brush; uint8 rop3; @@ -721,7 +721,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Stroke { - Path *path @c_ptr @marshall @nonnull; + Path *path @marshall @nonnull; LineAttr attr; Brush brush; uint16 fore_mode; @@ -744,7 +744,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Transparent { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; uint32 src_color; uint32 true_color; @@ -756,7 +756,7 @@ channel DisplayChannel : BaseChannel { struct AlphaBlnd { alpha_flags alpha_flags; uint8 alpha; - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; } data; } draw_alpha_blend; diff --git a/spice1.proto b/spice1.proto index 23ce9f1..d1efb55 100644 --- a/spice1.proto +++ b/spice1.proto @@ -382,7 +382,7 @@ struct Clip { clip_type type; switch (type) { case RECTS: - ClipRects *rects @outvar(cliprects) @c_ptr; + ClipRects *rects @outvar(cliprects); default: uint64 data @zero; } u @anon; @@ -421,7 +421,7 @@ struct BitmapData { case PAL_FROM_CACHE: uint64 palette_id; default: - Palette *palette @outvar(bitmap) @c_ptr; + Palette *palette @outvar(bitmap); } pal @anon; uint8 *data[image_size(8, stride, y)] @chunk; /* pointer to array, not array of pointers as in C */ } @ctype(SpiceBitmap); @@ -438,7 +438,7 @@ struct LZPLTData { case PAL_FROM_CACHE: uint64 palette_id; default: - Palette *palette @nonnull @outvar(lzplt) @c_ptr; + Palette *palette @nonnull @outvar(lzplt); } pal @anon; uint8 data[data_size] @nomarshal @chunk; }; @@ -466,7 +466,7 @@ struct Image { }; struct Pattern { - Image *pat @nonnull @c_ptr; + Image *pat @nonnull; Point pos; }; @@ -483,7 +483,7 @@ struct Brush { struct QMask { mask_flags flags; Point pos; - Image *bitmap @c_ptr; + Image *bitmap; }; struct LineAttr { @@ -493,7 +493,7 @@ struct LineAttr { uint8 style_nseg; fixed28_4 width @zero; fixed28_4 miter_limit @zero; - fixed28_4 *style[style_nseg] @c_ptr; + fixed28_4 *style[style_nseg]; }; struct RasterGlyphA1 { @@ -611,7 +611,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Opaque { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; Brush brush; ropd rop_descriptor; @@ -623,7 +623,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Copy { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -634,7 +634,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Blend { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; ropd rop_descriptor; image_scale_mode scale_mode; @@ -666,7 +666,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Rop3 { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; Brush brush; uint8 rop3; @@ -701,7 +701,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; struct Transparent { - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; uint32 src_color; uint32 true_color; @@ -713,7 +713,7 @@ channel DisplayChannel : BaseChannel { struct AlphaBlnd { int8 alpha_flags @virtual(0); uint8 alpha; - Image *src_bitmap @c_ptr; + Image *src_bitmap; Rect src_area; } data; } draw_alpha_blend; From a09be53ea82e238183b117641d99ab3f89248bd0 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 8 Jul 2010 15:44:15 +0200 Subject: [PATCH 058/147] Fix inclusion of common files, no need for common/ part --- python_modules/demarshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index e8afabb..96aa146 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1169,7 +1169,7 @@ def write_includes(writer): writer.writeln("#include ") writer.writeln("#include ") writer.writeln("#include ") - writer.writeln("#include ") + writer.writeln('#include "mem.h"') writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") From adce65685f52009d667719ed825fe5c6cc30b498 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 8 Jul 2010 18:26:37 +0200 Subject: [PATCH 059/147] Fix various misspellings letancy -> latency compund -> compound SpicedSubMessage -> SpiceSubMessage modifaiers -> modifiers massage -> message outgoiong -> outgoing AlphaBlnd -> AlphaBlend remoth -> remote modifires -> modifiers secore -> secure --- spice.proto | 2 +- spice1.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spice.proto b/spice.proto index f6aa34e..abb85cc 100644 --- a/spice.proto +++ b/spice.proto @@ -753,7 +753,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; - struct AlphaBlnd { + struct AlphaBlend { alpha_flags alpha_flags; uint8 alpha; Image *src_bitmap; diff --git a/spice1.proto b/spice1.proto index d1efb55..b4012bd 100644 --- a/spice1.proto +++ b/spice1.proto @@ -710,7 +710,7 @@ channel DisplayChannel : BaseChannel { message { DisplayBase base; - struct AlphaBlnd { + struct AlphaBlend { int8 alpha_flags @virtual(0); uint8 alpha; Image *src_bitmap; From 42c5286ca63f6ea9367a42bf3b073cd213541bd0 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 8 Jul 2010 19:22:39 +0200 Subject: [PATCH 060/147] Make distcheck work --- python_modules/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/Makefile.am b/python_modules/Makefile.am index 4b3c960..f304ec0 100644 --- a/python_modules/Makefile.am +++ b/python_modules/Makefile.am @@ -1,6 +1,6 @@ NULL = -PYTHON_MODULES = __init__.py codegen.py demarshal.py ptypes.py spice_parser.py +PYTHON_MODULES = __init__.py codegen.py demarshal.py marshal.py ptypes.py spice_parser.py EXTRA_DIST = $(PYTHON_MODULES) From 9e2e061bea9ca8fb690da29dca3d95da904b794a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 9 Jul 2010 16:52:36 +0200 Subject: [PATCH 061/147] Don't marshall bitmap data as pointer, instead send inline --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index abb85cc..ca01c12 100644 --- a/spice.proto +++ b/spice.proto @@ -439,7 +439,7 @@ struct BitmapData { default: Palette *palette @outvar(bitmap); } pal @anon; - uint8 *data[image_size(8, stride, y)] @chunk; /* pointer to array, not array of pointers as in C */ + uint8 data[image_size(8, stride, y)] @chunk @nomarshal; } @ctype(SpiceBitmap); struct BinaryData { From e6240ee0b07e2ab8e6aec5d9db4126ab7d4acf7c Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 09:45:45 +0200 Subject: [PATCH 062/147] codegen: Remove unused methos has_pointer() --- python_modules/ptypes.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 5e18aa3..e354454 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -130,9 +130,6 @@ class Type: _types.append(self) _types_by_name[self.name] = self - def has_pointer(self): - return False - def has_attr(self, name): return self.attributes.has_key(name) @@ -223,9 +220,6 @@ class TypeAlias(Type): return self.attributes["ctype"][0] return self.name - def has_pointer(self): - return self.the_type.has_pointer() - class EnumBaseType(Type): def is_enum(self): return isinstance(self, EnumType) @@ -474,9 +468,6 @@ class PointerType(Type): else: return "uint64_t" - def has_pointer(self): - return True - def contains_extra_size(self): return True @@ -570,9 +561,6 @@ class Member(Containee): def __repr__(self): return "%s (%s)" % (str(self.name), str(self.member_type)) - def has_pointer(self): - return self.member_type.has_pointer() - def get_num_pointers(self): return self.member_type.get_num_pointers() @@ -611,9 +599,6 @@ class SwitchCase: self.member = self.member.resolve(self) return self - def has_pointer(self): - return self.member.has_pointer() - def get_num_pointers(self): return self.member.get_num_pointers() @@ -706,12 +691,6 @@ class Switch(Containee): def contains_member(self, member): return False # TODO: Don't support switch deep member lookup yet - def has_pointer(self): - for c in self.cases: - if c.has_pointer(): - return True - return False - def get_num_pointers(self): count = 0 for c in self.cases: @@ -785,12 +764,6 @@ class ContainerType(Type): names = names + m.get_pointer_names(marshalled) return names - def has_pointer(self): - for m in self.members: - if m.has_pointer(): - return True - return False - def get_nw_offset(self, member, prefix = "", postfix = ""): fixed = self.get_fixed_nw_offset(member) v = [] From 373993f32c094296aa11ca61dec424dbed3083af Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 10:12:41 +0200 Subject: [PATCH 063/147] codegen: Pass member to SubMarshallingSource rather than name This way we can check attributes on the member. --- python_modules/marshal.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index df0c3b3..6b2d428 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -29,8 +29,8 @@ class MarshallingSource: def child_at_end(self, t): return RootMarshallingSource(self, t.c_type(), t.sizeof()) - def child_sub(self, member): - return SubMarshallingSource(self, member) + def child_sub(self, containee): + return SubMarshallingSource(self, containee) def declare(self, writer): return writer.optional_block(self.reuse_scope) @@ -84,18 +84,19 @@ class RootMarshallingSource(MarshallingSource): return writer.partial_block(scope) class SubMarshallingSource(MarshallingSource): - def __init__(self, parent_src, member): + def __init__(self, parent_src, containee): self.reuse_scope = None self.parent_src = parent_src self.base_var = parent_src.base_var - self.member = member + self.containee = containee + self.name = containee.name self.is_helper = False def get_self_ref(self): - return "&%s" % self.parent_src.get_ref(self.member) + return "&%s" % self.parent_src.get_ref(self.name) def get_ref(self, member): - return self.parent_src.get_ref(self.member) + "." + member + return self.parent_src.get_ref(self.name) + "." + member def write_marshal_ptr_function(writer, target_type): if target_type.is_array(): @@ -263,9 +264,9 @@ def write_switch_marshaller(writer, container, switch, src, scope): src2 = src else: if t.is_struct(): - src2 = src.child_sub(switch.name + "." + m.name) + src2 = src.child_sub(switch).child_sub(m) else: - src2 = src.child_sub(switch.name) + src2 = src.child_sub(switch) src2.reuse_scope = block if t.is_struct(): @@ -331,7 +332,7 @@ def write_member_marshaller(writer, container, member, src, scope): if member.has_end_attr(): src2 = src.child_at_end(t) else: - src2 = src.child_sub(member.name) + src2 = src.child_sub(member) writer.comment(member.name) write_container_marshaller(writer, t, src2) else: From d9629ca4e700cb35b77579bde4ab78ad7adbb30a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 14:10:16 +0200 Subject: [PATCH 064/147] codegen: Various cleanups Remove all uses of @end in the marshaller, instead just using the C struct array-at-end-of-struct. To make this work we also remove all use of @end for switches (making them C unions). We drop the zero member of the notify message so that we can avoid this use of @end for a primitive in the marshaller (plus its useless to send over the wire). We change the offsets and stuff in the migration messages to real pointers. --- python_modules/demarshal.py | 34 ++++++++++++------- python_modules/marshal.py | 68 +++++++++++++------------------------ spice.proto | 19 ++++------- spice1.proto | 16 ++++----- 4 files changed, 59 insertions(+), 78 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 96aa146..63f952b 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -646,8 +646,11 @@ class SubDemarshallingDestination(DemarshallingDestination): return self.parent_dest.get_ref(self.member) + "." + member # Note: during parsing, byte_size types have been converted to count during validation -def read_array_len(writer, prefix, array, dest, scope): - nelements = "%s__nelements" % prefix +def read_array_len(writer, prefix, array, dest, scope, is_ptr): + if is_ptr: + nelements = "%s__array__nelements" % prefix + else: + nelements = "%s__nelements" % prefix if dest.is_toplevel(): return nelements # Already there for toplevel, need not recalculate element_type = array.element_type @@ -716,8 +719,8 @@ def write_switch_parser(writer, container, switch, dest, scope): writer.assign(dest2.get_ref(m.name), "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums elif t.is_array(): - nelements = read_array_len(writer, m.name, t, dest, block) - write_array_parser(writer, nelements, t, dest, block) + nelements = read_array_len(writer, m.name, t, dest, block, False) + write_array_parser(writer, m, nelements, t, dest2, block) else: writer.todo("Can't handle type %s" % m.member_type) @@ -759,7 +762,7 @@ def write_parse_ptr_function(writer, target_type): dest.is_helper = True dest.reuse_scope = scope if target_type.is_array(): - write_array_parser(writer, "this_ptr_info->nelements", target_type, dest, scope) + write_array_parser(writer, None, "this_ptr_info->nelements", target_type, dest, scope) else: write_container_parser(writer, target_type, dest) @@ -777,14 +780,17 @@ def write_parse_ptr_function(writer, target_type): return parse_function -def write_array_parser(writer, nelements, array, dest, scope): +def write_array_parser(writer, member, nelements, array, dest, scope): is_byte_size = array.is_bytes_length() element_type = array.element_type if element_type == ptypes.uint8 or element_type == ptypes.int8: - writer.statement("memcpy(end, in, %s)" % (nelements)) + if not member or member.has_attr("end"): + writer.statement("memcpy(end, in, %s)" % (nelements)) + writer.increment("end", nelements) + else: + writer.statement("memcpy(%s, in, %s)" % (dest.get_ref(member.name), nelements)) writer.increment("in", nelements) - writer.increment("end", nelements) else: if array.has_attr("ptr_array"): scope.variable_def("void **", "ptr_array") @@ -817,7 +823,7 @@ def write_parse_pointer(writer, t, at_end, dest, member_name, scope): else: writer.assign("ptr_info[n_ptr].dest", "(void **)&%s" % dest.get_ref(member_name)) if target_type.is_array(): - nelements = read_array_len(writer, member_name, target_type, dest, scope) + nelements = read_array_len(writer, member_name, target_type, dest, scope, True) writer.assign("ptr_info[n_ptr].nelements", nelements) writer.statement("n_ptr++") @@ -836,7 +842,7 @@ def write_member_parser(writer, container, member, dest, scope): if t.is_pointer(): if member.has_attr("chunk"): assert(t.target_type.is_array()) - nelements = read_array_len(writer, member.name, t.target_type, dest, scope) + nelements = read_array_len(writer, member.name, t.target_type, dest, scope, True) writer.comment("Reuse data from network message as chunk").newline() scope.variable_def("SpiceChunks *", "chunks"); writer.assign("chunks", "(SpiceChunks *)end") @@ -866,7 +872,7 @@ def write_member_parser(writer, container, member, dest, scope): writer.assign(dest_var, "consume_%s(&in)" % (t.primitive_type())) #TODO validate e.g. flags and enums elif t.is_array(): - nelements = read_array_len(writer, member.name, t, dest, scope) + nelements = read_array_len(writer, member.name, t, dest, scope, False) if member.has_attr("chunk") and t.element_type.is_fixed_nw_size() and t.element_type.get_fixed_nw_size() == 1: writer.comment("use array as chunk").newline() @@ -892,7 +898,7 @@ def write_member_parser(writer, container, member, dest, scope): else: writer.increment("in", "%s" % (nelements)) else: - write_array_parser(writer, nelements, t, dest, scope) + write_array_parser(writer, member, nelements, t, dest, scope) elif t.is_struct(): if member.has_end_attr(): dest2 = dest.child_at_end(writer, t) @@ -915,7 +921,9 @@ def write_container_parser(writer, container, dest): writer.end_block(newline=False) writer.begin_block(" else") # TODO: This is not right for fields that don't exist in the struct - if m.member_type.is_primitive(): + if m.has_attr("zero"): + pass + elif m.member_type.is_primitive(): writer.assign(dest.get_ref(m.name), "0") elif m.is_fixed_sizeof(): writer.statement("memset ((char *)&%s, 0, %s)" % (dest.get_ref(m.name), m.sizeof())) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 6b2d428..cf6ad08 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -49,8 +49,8 @@ class RootMarshallingSource(MarshallingSource): self.base_var = "src" self.c_type = c_type self.sizeof = sizeof - self.pointer = pointer # None == at "end" - self.update_end = False + self.pointer = pointer + assert pointer != None def get_self_ref(self): return self.base_var @@ -69,13 +69,7 @@ class RootMarshallingSource(MarshallingSource): if not self.reuse_scope: scope.newline() - if self.pointer: - writer.assign(self.base_var, "(%s *)%s" % (self.c_type, self.pointer)) - if self.update_end: - writer.assign("end", "((uint8_t *)%s) + %s" % (self.base_var, self.sizeof)) - else: - writer.assign(self.base_var, "(%s *)end" % self.c_type) - writer.increment("end", "%s" % self.sizeof) + writer.assign(self.base_var, "(%s *)%s" % (self.c_type, self.pointer)) writer.newline() if self.reuse_scope: @@ -119,18 +113,16 @@ def write_marshal_ptr_function(writer, target_type): writer.header = header writer.out_prefix = "" if target_type.is_array(): - scope = writer.function(marshal_function, "SPICE_GNUC_UNUSED static void *", "SpiceMarshaller *m, %s_t *ptr, int count" % target_type.element_type.primitive_type() + names_args) + scope = writer.function(marshal_function, "SPICE_GNUC_UNUSED static void", "SpiceMarshaller *m, %s_t *ptr, int count" % target_type.element_type.primitive_type() + names_args) else: - scope = writer.function(marshal_function, "void *", "SpiceMarshaller *m, %s *ptr" % target_type.c_type() + names_args) - header.writeln("void *" + marshal_function + "(SpiceMarshaller *m, %s *msg" % target_type.c_type() + names_args + ");") - scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") + scope = writer.function(marshal_function, "void", "SpiceMarshaller *m, %s *ptr" % target_type.c_type() + names_args) + header.writeln("void " + marshal_function + "(SpiceMarshaller *m, %s *msg" % target_type.c_type() + names_args + ");") scope.variable_def("SPICE_GNUC_UNUSED SpiceMarshaller *", "m2") for n in names: writer.assign("*%s_out" % n, "NULL") writer.newline() - writer.assign("end", "(uint8_t *)(ptr+1)") if target_type.is_struct(): src = RootMarshallingSource(None, target_type.c_type(), target_type.sizeof(), "ptr") @@ -143,8 +135,6 @@ def write_marshal_ptr_function(writer, target_type): else: writer.todo("Unsuppored pointer marshaller type") - writer.statement("return end") - writer.end_block() return marshal_function @@ -172,9 +162,9 @@ def get_array_size(array, container_src): elif array.is_bytes_length(): return container_src.get_ref(array.size[2]) else: - raise NotImplementedError("TODO array size type not handled yet") + raise NotImplementedError("TODO array size type not handled yet: %s" % array) -def write_array_marshaller(writer, at_end, member, array, container_src, scope): +def write_array_marshaller(writer, member, array, container_src, scope): element_type = array.element_type if array.is_remaining_length(): @@ -196,8 +186,7 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): if array.has_attr("ptr_array"): element = "*" + element - if not at_end: - writer.assign(element_array, container_src.get_ref(member.name)) + writer.assign(element_array, container_src.get_ref(member.name)) if is_byte_size: size_start_var = "%s__size_start" % member.name @@ -206,23 +195,16 @@ def write_array_marshaller(writer, at_end, member, array, container_src, scope): with writer.index() as index: with writer.for_loop(index, nelements) as array_scope: - if at_end: - writer.assign(element, "(%s *)end" % element_type.c_type()) - writer.increment("end", element_type.sizeof()) - if element_type.is_primitive(): writer.statement("spice_marshaller_add_%s(m, *%s)" % (element_type.primitive_type(), element)) elif element_type.is_struct(): src2 = RootMarshallingSource(container_src, element_type.c_type(), element_type.sizeof(), element) - if array.is_extra_size(): - src2.update_end = True src2.reuse_scope = array_scope write_container_marshaller(writer, element_type, src2) else: writer.todo("array element unhandled type").newline() - if not at_end: - writer.statement("%s++" % element_array) + writer.statement("%s++" % element_array) if is_byte_size: size_var = member.container.lookup_member(array.size[1]) @@ -235,12 +217,15 @@ def write_pointer_marshaller(writer, member, src): ptr_func = write_marshal_ptr_function(writer, t.target_type) submarshaller = "spice_marshaller_get_ptr_submarshaller(m, %d)" % (1 if member.get_fixed_nw_size() == 8 else 0) if member.has_attr("marshall"): + rest_args = "" + if t.target_type.is_array(): + rest_args = ", %s" % get_array_size(t.target_type, src) writer.assign("m2", submarshaller) if t.has_attr("nonnull"): - writer.statement("%s(m2, %s)" % (ptr_func, src.get_ref(member.name))) + writer.statement("%s(m2, %s%s)" % (ptr_func, src.get_ref(member.name), rest_args)) else: with writer.if_block("%s != NULL" % src.get_ref(member.name)) as block: - writer.statement("%s(m2, %s)" % (ptr_func, src.get_ref(member.name))) + writer.statement("%s(m2, %s%s)" % (ptr_func, src.get_ref(member.name), rest_args)) else: writer.assign("*%s_out" % (writer.out_prefix + member.name), submarshaller) @@ -258,10 +243,11 @@ def write_switch_marshaller(writer, container, switch, src, scope): writer.out_prefix = "%s_%s" % (m.attributes["outvar"][0], writer.out_prefix) with writer.if_block(check, not first, False) as block: t = m.member_type - if switch.has_end_attr(): - src2 = src.child_at_end(m.member_type) - elif switch.has_attr("anon"): - src2 = src + if switch.has_attr("anon"): + if t.is_struct(): + src2 = src.child_sub(m) + else: + src2 = src else: if t.is_struct(): src2 = src.child_sub(switch).child_sub(m) @@ -280,7 +266,7 @@ def write_switch_marshaller(writer, container, switch, src, scope): writer.statement("spice_marshaller_add_%s(m, %s)" % (t.primitive_type(), src2.get_ref(m.name))) #TODO validate e.g. flags and enums elif t.is_array(): - write_array_marshaller(writer, switch.has_end_attr(), m, t, src, scope) + write_array_marshaller(writer, m, t, src2, scope) else: writer.todo("Can't handle type %s" % m.member_type) @@ -321,18 +307,12 @@ def write_member_marshaller(writer, container, member, src, scope): scope.variable_def("void *", var) writer.statement("%s = spice_marshaller_add_%s(m, %s)" % (var, t.primitive_type(), 0)) - elif member.has_end_attr(): - writer.statement("spice_marshaller_add_%s(m, *(%s_t *)end)" % (t.primitive_type(), t.primitive_type())) - writer.increment("end", t.sizeof()) else: writer.statement("spice_marshaller_add_%s(m, %s)" % (t.primitive_type(), src.get_ref(member.name))) elif t.is_array(): - write_array_marshaller(writer, member.has_end_attr(), member, t, src, scope) + write_array_marshaller(writer, member, t, src, scope) elif t.is_struct(): - if member.has_end_attr(): - src2 = src.child_at_end(t) - else: - src2 = src.child_sub(member) + src2 = src.child_sub(member) writer.comment(member.name) write_container_marshaller(writer, t, src2) else: @@ -364,7 +344,6 @@ def write_message_marshaller(writer, message, is_server, private): scope = writer.function(function_name, "static void" if private else "void", "SpiceMarshaller *m, %s *msg" % message.c_type() + names_args) - scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "end") scope.variable_def("SPICE_GNUC_UNUSED SpiceMarshaller *", "m2") for n in names: @@ -373,7 +352,6 @@ def write_message_marshaller(writer, message, is_server, private): src = RootMarshallingSource(None, message.c_type(), message.sizeof(), "msg") src.reuse_scope = scope - writer.assign("end", "(uint8_t *)(msg+1)") write_container_marshaller(writer, message, src) writer.end_block() diff --git a/spice.proto b/spice.proto index ca01c12..fbafd82 100644 --- a/spice.proto +++ b/spice.proto @@ -131,7 +131,6 @@ channel BaseChannel { uint32 what; /* error_code/warn_code/info_code */ uint32 message_len; uint8 message[message_len] @end @nomarshal; - uint8 zero @end @ctype(uint8_t) @nomarshal; } notify; client: @@ -166,13 +165,11 @@ channel MainChannel : BaseChannel { message { uint16 port; uint16 sport; - uint32 host_offset; uint32 host_size; + uint8 *host_data[host_size] @zero_terminated @marshall @nonnull; pubkey_type pub_key_type; - uint32 pub_key_offset; uint32 pub_key_size; - uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; - uint8 pub_key_data[pub_key_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; + uint8 *pub_key_data[pub_key_size] @zero_terminated @marshall @nonnull; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; @@ -217,12 +214,10 @@ channel MainChannel : BaseChannel { message { uint16 port; uint16 sport; - uint32 host_offset; uint32 host_size; - uint32 cert_subject_offset; + uint8 *host_data[host_size] @zero_terminated @marshall; uint32 cert_subject_size; - uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated; - uint8 cert_subject_data[cert_subject_size] @end @ctype(uint8_t) @zero_terminated; + uint8 *cert_subject_data[cert_subject_size] @zero_terminated @marshall; } @ctype(SpiceMsgMainMigrationSwitchHost) migrate_switch_host; client: @@ -994,8 +989,8 @@ struct TunnelIpInfo { tunnel_ip_type type; switch (type) { case IPv4: - uint8 ipv4[4] @ctype(uint8_t); - } u @end @nomarshal; + uint8 ipv4[4]; + } u; } @ctype(SpiceMsgTunnelIpInfo); channel TunnelChannel : BaseChannel { @@ -1049,7 +1044,7 @@ channel TunnelChannel : BaseChannel { switch (type) { case IPP: TunnelIpInfo ip @ctype(SpiceMsgTunnelIpInfo); - } u @end; + } u; } @ctype(SpiceMsgcTunnelAddGenericService) service_add = 101; message { diff --git a/spice1.proto b/spice1.proto index b4012bd..e38a214 100644 --- a/spice1.proto +++ b/spice1.proto @@ -166,13 +166,13 @@ channel MainChannel : BaseChannel { message { uint16 port; uint16 sport; - uint32 host_offset; + uint32 host_offset @zero; uint32 host_size; pubkey_type pub_key_type @minor(2); - uint32 pub_key_offset @minor(2); + uint32 pub_key_offset @minor(2) @zero; uint32 pub_key_size @minor(2); - uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated @nomarshal; - uint8 pub_key_data[pub_key_size] @minor(2) @end @ctype(uint8_t) @zero_terminated @nomarshal; + uint8 host_data[host_size] @as_ptr @zero_terminated; + uint8 pub_key_data[pub_key_size] @minor(2) @as_ptr @zero_terminated; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; @@ -217,12 +217,12 @@ channel MainChannel : BaseChannel { message { uint16 port; uint16 sport; - uint32 host_offset; + uint32 host_offset @zero; uint32 host_size; - uint32 cert_subject_offset; + uint32 cert_subject_offset @zero; uint32 cert_subject_size; - uint8 host_data[host_size] @end @ctype(uint8_t) @zero_terminated; - uint8 cert_subject_data[cert_subject_size] @end @ctype(uint8_t) @zero_terminated; + uint8 host_data[host_size] @as_ptr @zero_terminated; + uint8 cert_subject_data[cert_subject_size] @as_ptr @zero_terminated; } @ctype(SpiceMsgMainMigrationSwitchHost) migrate_switch_host; client: From f008b7605df19ab5deb8f4f383837bef9416d7a1 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 15:47:40 +0200 Subject: [PATCH 065/147] codegen: Allow @to_ptr to make inline structs demarshal as pointers --- python_modules/demarshal.py | 32 ++++++++++++++++++++++++++------ python_modules/marshal.py | 10 ++++++++-- python_modules/ptypes.py | 4 +++- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 63f952b..4aba7b0 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -161,6 +161,7 @@ def write_validate_switch_member(writer, container, switch_member, scope, parent with writer.if_block(check, not first, False) as if_scope: item.type = c.member.member_type item.subprefix = item.prefix + "_" + m.name + item.member = c.member all_as_extra_size = m.is_extra_size() and want_extra_size if not want_mem_size and all_as_extra_size and not scope.variable_defined(item.mem_size()): @@ -459,6 +460,8 @@ def write_validate_primitive_item(writer, container, item, scope, parent_scope, def write_validate_item(writer, container, item, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): + if item.member and item.member.has_attr("to_ptr"): + want_nw_size = True if item.type.is_pointer(): write_validate_pointer_item(writer, container, item, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size) @@ -474,6 +477,11 @@ def write_validate_item(writer, container, item, scope, parent_scope, start, else: writer.todo("Implement validation of %s" % item.type) + if item.member and item.member.has_attr("to_ptr"): + saved_size = "%s__saved_size" % item.member.name + writer.add_function_variable("uint32_t", saved_size) + writer.assign(saved_size, item.nw_size()) + def write_validate_member(writer, container, member, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): if member.has_attr("virtual"): @@ -708,10 +716,12 @@ def write_switch_parser(writer, container, switch, dest, scope): dest2 = dest.child_sub(switch.name) dest2.reuse_scope = block - if t.is_struct(): - write_container_parser(writer, t, dest2) + if m.has_attr("to_ptr"): + write_parse_to_pointer(writer, t, False, dest2, m.name, block) elif t.is_pointer(): write_parse_pointer(writer, t, False, dest2, m.name, block) + elif t.is_struct(): + write_container_parser(writer, t, dest2) elif t.is_primitive(): if m.has_attr("zero"): writer.statement("consume_%s(&in)" % (t.primitive_type())) @@ -813,9 +823,8 @@ def write_array_parser(writer, member, nelements, array, dest, scope): writer.comment("Align ptr_array element to 4 bytes").newline() writer.assign("end", "(uint8_t *)SPICE_ALIGN((size_t)end, 4)") -def write_parse_pointer(writer, t, at_end, dest, member_name, scope): - target_type = t.target_type - writer.assign("ptr_info[n_ptr].offset", "consume_%s(&in)" % t.primitive_type()) +def write_parse_pointer_core(writer, target_type, offset, at_end, dest, member_name, scope): + writer.assign("ptr_info[n_ptr].offset", offset) writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) if at_end: writer.assign("ptr_info[n_ptr].dest", "(void **)end") @@ -828,6 +837,15 @@ def write_parse_pointer(writer, t, at_end, dest, member_name, scope): writer.statement("n_ptr++") +def write_parse_pointer(writer, t, at_end, dest, member_name, scope): + write_parse_pointer_core(writer, t.target_type, "consume_%s(&in)" % t.primitive_type(), + at_end, dest, member_name, scope) + +def write_parse_to_pointer(writer, t, at_end, dest, member_name, scope): + write_parse_pointer_core(writer, t, "in - start", + at_end, dest, member_name, scope) + writer.increment("in", "%s__saved_size" % member_name) + def write_member_parser(writer, container, member, dest, scope): if member.has_attr("virtual"): writer.assign(dest.get_ref(member.name), member.attributes["virtual"][0]) @@ -839,7 +857,9 @@ def write_member_parser(writer, container, member, dest, scope): t = member.member_type - if t.is_pointer(): + if member.has_attr("to_ptr"): + write_parse_to_pointer(writer, t, member.has_end_attr(), dest, member.name, scope) + elif t.is_pointer(): if member.has_attr("chunk"): assert(t.target_type.is_array()) nelements = read_array_len(writer, member.name, t.target_type, dest, scope, True) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index cf6ad08..6b894c1 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -87,10 +87,16 @@ class SubMarshallingSource(MarshallingSource): self.is_helper = False def get_self_ref(self): - return "&%s" % self.parent_src.get_ref(self.name) + if self.containee.has_attr("to_ptr"): + return "%s" % self.parent_src.get_ref(self.name) + else: + return "&%s" % self.parent_src.get_ref(self.name) def get_ref(self, member): - return self.parent_src.get_ref(self.name) + "." + member + if self.containee.has_attr("to_ptr"): + return self.parent_src.get_ref(self.name) + "->" + member + else: + return self.parent_src.get_ref(self.name) + "." + member def write_marshal_ptr_function(writer, target_type): if target_type.is_array(): diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index e354454..59ed897 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -536,7 +536,7 @@ class Member(Containee): return self.member_type.is_fixed_sizeof() def is_extra_size(self): - return self.has_end_attr() or self.member_type.is_extra_size() + return self.has_end_attr() or self.has_attr("to_ptr") or self.member_type.is_extra_size() def is_fixed_nw_size(self): if self.has_attr("virtual"): @@ -562,6 +562,8 @@ class Member(Containee): return "%s (%s)" % (str(self.name), str(self.member_type)) def get_num_pointers(self): + if self.has_attr("to_ptr"): + return 1 return self.member_type.get_num_pointers() def get_pointer_names(self, marshalled): From f3d92c736b5a2d25f6a881e7234e798b7e7ae7e5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 15:48:20 +0200 Subject: [PATCH 066/147] Send Clip.rects inline rather than using a pointer --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index fbafd82..f24445b 100644 --- a/spice.proto +++ b/spice.proto @@ -395,7 +395,7 @@ struct Clip { clip_type type; switch (type) { case RECTS: - ClipRects *rects @outvar(cliprects) @marshall @nonnull; + ClipRects rects @outvar(cliprects) @to_ptr; } u @anon; }; From 4702feb5b1958eeef9b4414a2fb7cac259394a52 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 16:27:42 +0200 Subject: [PATCH 067/147] Don't send CursorHeader if cursor_flags is NONE --- python_modules/demarshal.py | 5 ++++- python_modules/ptypes.py | 4 ++-- python_modules/spice_parser.py | 2 +- spice.proto | 5 ++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 4aba7b0..f259ead 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -708,7 +708,10 @@ def write_switch_parser(writer, container, switch, dest, scope): if switch.has_end_attr(): dest2 = dest.child_at_end(writer, m.member_type) elif switch.has_attr("anon"): - dest2 = dest + if t.is_struct() and not m.has_attr("to_ptr"): + dest2 = dest.child_sub(m.name) + else: + dest2 = dest else: if t.is_struct(): dest2 = dest.child_sub(switch.name + "." + m.name) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 59ed897..0ae57ec 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -591,9 +591,9 @@ class SwitchCase: if v == None: return "1" elif var_type.is_enum(): - checks.append("%s == %s" % (var_cname, var_type.c_enumname_by_name(v))) + checks.append("%s == %s" % (var_cname, var_type.c_enumname_by_name(v[1]))) else: - checks.append("(%s & %s)" % (var_cname, var_type.c_enumname_by_name(v))) + checks.append("%s(%s & %s)" % (v[0], var_cname, var_type.c_enumname_by_name(v[1]))) return " || ".join(checks) def resolve(self, container): diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index ac2da8d..43e930c 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -93,7 +93,7 @@ def SPICE_BNF(): variableDef = Group(typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi) \ .setParseAction(parseVariableDef) - switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | case_.suppress() + identifier + colon)) + variableDef) \ + switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | Group(case_.suppress() + Optional("!", default="") + identifier) + colon)) + variableDef) \ .setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) switchBody = Group(switch_ + lparen + delimitedList(identifier,delim='.', combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \ .setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) diff --git a/spice.proto b/spice.proto index f24445b..3f7c30c 100644 --- a/spice.proto +++ b/spice.proto @@ -874,7 +874,10 @@ struct CursorHeader { struct Cursor { cursor_flags flags; - CursorHeader header; + switch (flags) { + case !NONE: + CursorHeader header; + } u @anon; uint8 data[] @as_ptr(data_size); }; From 12bc82da5cc172df06cf1f5fbddd492e93e18fe6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jul 2010 20:24:25 +0200 Subject: [PATCH 068/147] Fix uninitialized variable warnings --- python_modules/demarshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index f259ead..a4345d4 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -479,7 +479,7 @@ def write_validate_item(writer, container, item, scope, parent_scope, start, if item.member and item.member.has_attr("to_ptr"): saved_size = "%s__saved_size" % item.member.name - writer.add_function_variable("uint32_t", saved_size) + writer.add_function_variable("uint32_t", saved_size + " = 0") writer.assign(saved_size, item.nw_size()) def write_validate_member(writer, container, member, parent_scope, start, From 7861ba0921cae2a8616fa625693e3a887da43f94 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 20 Jul 2010 11:33:57 +0200 Subject: [PATCH 069/147] Don't send padding over the network with video data --- spice.proto | 2 -- spice1.proto | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/spice.proto b/spice.proto index 3f7c30c..3c0911d 100644 --- a/spice.proto +++ b/spice.proto @@ -621,9 +621,7 @@ channel DisplayChannel : BaseChannel { uint32 id; uint32 multi_media_time; uint32 data_size; - uint32 pad_size; uint8 data[data_size] @end @nomarshal; - uint8 padding[pad_size] @end @ctype(uint8_t) @nomarshal; /* Uhm, why are we sending padding over network? */ } stream_data; message { diff --git a/spice1.proto b/spice1.proto index e38a214..ebb2d6f 100644 --- a/spice1.proto +++ b/spice1.proto @@ -583,9 +583,9 @@ channel DisplayChannel : BaseChannel { uint32 id; uint32 multi_media_time; uint32 data_size; - uint32 pad_size; + uint32 pad_size @zero; uint8 data[data_size] @end @nomarshal; - uint8 padding[pad_size] @end @ctype(uint8_t) @nomarshal; /* Uhm, why are we sending padding over network? */ + /* Ignore: uint8 padding[pad_size] */ } stream_data; message { From aae971c6abff7deacc9ffcde8aab87ded08b24c3 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 20 Jul 2010 11:34:23 +0200 Subject: [PATCH 070/147] demarshaller: Fix palette marshalling It turns out that using base + sizeof(struct) is not a good way to access an array at the end of a struct. For SpicePalette sizeof is 16, but offset of ents is 12. Using this calculation in the demarshaller breaks things badly, so now we use the actual array member. --- python_modules/demarshal.py | 50 +++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index a4345d4..c588f98 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -797,29 +797,47 @@ def write_array_parser(writer, member, nelements, array, dest, scope): is_byte_size = array.is_bytes_length() element_type = array.element_type - if element_type == ptypes.uint8 or element_type == ptypes.int8: - if not member or member.has_attr("end"): - writer.statement("memcpy(end, in, %s)" % (nelements)) - writer.increment("end", nelements) - else: - writer.statement("memcpy(%s, in, %s)" % (dest.get_ref(member.name), nelements)) - writer.increment("in", nelements) + if member: + array_start = dest.get_ref(member.name) + at_end = member.has_attr("end") + else: + array_start = "end" + at_end = True + + if element_type == ptypes.uint8 or element_type == ptypes.int8: + writer.statement("memcpy(%s, in, %s)" % (array_start, nelements)) + writer.increment("in", nelements) + if at_end: + writer.increment("end", nelements) else: - if array.has_attr("ptr_array"): - scope.variable_def("void **", "ptr_array") - scope.variable_def("int", "ptr_array_index") - writer.assign("ptr_array_index", 0) - writer.assign("ptr_array", "(void **)end") - writer.increment("end", "sizeof(void *) * %s" % nelements) with writer.index() as index: + if member: + array_pos = "%s[%s]" % (array_start, index) + else: + array_pos = "*(%s *)end" % (element_type.c_type()) + + if array.has_attr("ptr_array"): + scope.variable_def("void **", "ptr_array") + scope.variable_def("int", "ptr_array_index") + writer.assign("ptr_array_index", 0) + writer.assign("ptr_array", "(void **)%s" % array_start) + writer.increment("end", "sizeof(void *) * %s" % nelements) + array_start = "end" + array_pos = "*(%s *)end" % (element_type.c_type()) + at_end = True + with writer.for_loop(index, nelements) as array_scope: if array.has_attr("ptr_array"): writer.statement("ptr_array[ptr_array_index++] = end") if element_type.is_primitive(): - writer.statement("*(%s *)end = consume_%s(&in)" % (element_type.c_type(), element_type.primitive_type())) - writer.increment("end", element_type.sizeof()) + writer.statement("%s = consume_%s(&in)" % (array_pos, element_type.primitive_type())) + if at_end: + writer.increment("end", element_type.sizeof()) else: - dest2 = dest.child_at_end(writer, element_type) + if at_end: + dest2 = dest.child_at_end(writer, element_type) + else: + dest2 = RootDemarshallingDestination(dest, element_type.c_type(), element_type.c_type(), array_pos) dest2.reuse_scope = array_scope write_container_parser(writer, element_type, dest2) if array.has_attr("ptr_array"): From aa7a086933c8b03d2f7f78874342a53719a24fa5 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Thu, 29 Jul 2010 09:03:15 -0400 Subject: [PATCH 071/147] support python 2.5.4+ for marshaller/demarshallers Patch adds a "from __future__" import that doesn't affect newer python's but allows python 2.5.4 to run the code (tested under scratchbox, n900 build environment) --- python_modules/codegen.py | 1 + python_modules/demarshal.py | 1 + python_modules/marshal.py | 1 + 3 files changed, 3 insertions(+) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index af6636b..03c67e6 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -1,3 +1,4 @@ +from __future__ import with_statement from cStringIO import StringIO def camel_to_underscores(s, upper = False): diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index c588f98..cbe3599 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import ptypes import codegen diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 6b894c1..9ee1466 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import ptypes import codegen From 005ecaa7d6a085d7210140553d680341cf72a227 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 4 Oct 2010 22:02:41 +0200 Subject: [PATCH 072/147] spice codegen: fix copy-o, no such variable value --- python_modules/ptypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 0ae57ec..9c4b7de 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -240,7 +240,7 @@ class EnumBaseType(Type): def c_enumname_by_name(self, name): if self.has_attr("prefix"): - return self.attributes["prefix"][0] + self.names[value] + return self.attributes["prefix"][0] + name return codegen.prefix_underscore_upper(self.name.upper(), name) def is_primitive(self): From 8a40c89280c426d0fb95c7d9d4845b4a5feb6c37 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 13 Sep 2010 20:02:42 +0200 Subject: [PATCH 073/147] smartcard: add to spice.proto --- spice.proto | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spice.proto b/spice.proto index 3c0911d..4eeb159 100644 --- a/spice.proto +++ b/spice.proto @@ -1084,6 +1084,13 @@ channel TunnelChannel : BaseChannel { } @ctype(SpiceMsgcTunnelSocketTokens) socket_token; }; +channel SmartcardChannel : BaseChannel { +server: + Data data = 101; +client: + Data data = 101; +}; + protocol Spice { MainChannel main = 1; DisplayChannel display; @@ -1092,4 +1099,5 @@ protocol Spice { PlaybackChannel playback; RecordChannel record; TunnelChannel tunnel; + SmartcardChannel smartcard; }; From 11269608a77b22df32ac1760b1df261c2b407af7 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 6 Dec 2010 18:02:34 +0200 Subject: [PATCH 074/147] mingw32 build: python_modules/marshal: use unsigned for for_loop index variable --- python_modules/marshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 9ee1466..a82df98 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -120,7 +120,7 @@ def write_marshal_ptr_function(writer, target_type): writer.header = header writer.out_prefix = "" if target_type.is_array(): - scope = writer.function(marshal_function, "SPICE_GNUC_UNUSED static void", "SpiceMarshaller *m, %s_t *ptr, int count" % target_type.element_type.primitive_type() + names_args) + scope = writer.function(marshal_function, "SPICE_GNUC_UNUSED static void", "SpiceMarshaller *m, %s_t *ptr, unsigned count" % target_type.element_type.primitive_type() + names_args) else: scope = writer.function(marshal_function, "void", "SpiceMarshaller *m, %s *ptr" % target_type.c_type() + names_args) header.writeln("void " + marshal_function + "(SpiceMarshaller *m, %s *msg" % target_type.c_type() + names_args + ");") From d6f198b3f433100a7a212be03b059876ed33e598 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 24 Jan 2011 22:07:55 +0200 Subject: [PATCH 075/147] codegen: avoid creating out if not used (fix gcc 4.6.0 warning) --- python_modules/demarshal.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index cbe3599..48551c0 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1047,9 +1047,11 @@ def write_msg_parser(writer, message): writer.assign("end", "data + %s" % (msg_sizeof)) writer.assign("in", "start").newline() - dest = RootDemarshallingDestination(None, msg_type, msg_sizeof, "data") - dest.reuse_scope = parent_scope - write_container_parser(writer, message, dest) + # avoid defined and assigned but not used warnings of gcc 4.6.0+ + if message.is_extra_size() or not message.is_fixed_nw_size() or message.get_fixed_nw_size() > 0: + dest = RootDemarshallingDestination(None, msg_type, msg_sizeof, "data") + dest.reuse_scope = parent_scope + write_container_parser(writer, message, dest) writer.newline() writer.statement("assert(in <= message_end)") From 3a6de6d6fb0362df9e78fa64942a45c02bdabe53 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 24 Jan 2011 23:32:43 +0200 Subject: [PATCH 076/147] demarshaller/marshaller fix gcc 4.6.0 python_modules/demarshal.py and marshal.py fixes for gcc 4.6.0 warning about set but unused variables. The fixes disable creating of variables mem_size when they are not used (demarshall) and declaring a src variable when the message doesn't use it (marshal). You need to touch *.proto after applying this (should add a Makefile dependency). --- python_modules/demarshal.py | 22 ++++++++++++++++------ python_modules/marshal.py | 8 +++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 48551c0..9d3b1e4 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -251,14 +251,18 @@ def write_validate_pointer_item(writer, container, item, scope, parent_scope, st array_item = ItemInfo(target_type, "%s__array" % item.prefix, start) scope.variable_def("uint32_t", array_item.nw_size()) - scope.variable_def("uint32_t", array_item.mem_size()) + # don't create a variable that isn't used, fixes -Werror=unused-but-set-variable + need_mem_size = want_mem_size or ( + want_extra_size and not item.member.has_attr("chunk") + and not target_type.is_cstring_length()) + if need_mem_size: + scope.variable_def("uint32_t", array_item.mem_size()) if target_type.is_cstring_length(): writer.assign(array_item.nw_size(), "spice_strnlen((char *)message_start + %s, message_end - (message_start + %s))" % (v, v)) writer.error_check("*(message_start + %s + %s) != 0" % (v, array_item.nw_size())) - writer.assign(array_item.mem_size(), array_item.nw_size()) else: write_validate_array_item(writer, container, array_item, scope, parent_scope, start, - True, True, False) + True, want_mem_size=need_mem_size, want_extra_size=False) writer.error_check("message_start + %s + %s > message_end" % (v, array_item.nw_size())) if want_extra_size: @@ -524,7 +528,7 @@ def write_validate_member(writer, container, member, parent_scope, start, def write_validate_container(writer, prefix, container, start, parent_scope, want_nw_size, want_mem_size, want_extra_size): for m in container.members: sub_want_nw_size = want_nw_size and not m.is_fixed_nw_size() - sub_want_mem_size = m.is_extra_size() + sub_want_mem_size = m.is_extra_size() and want_mem_size sub_want_extra_size = not m.is_extra_size() and m.contains_extra_size() defs = ["size_t"] @@ -1007,6 +1011,9 @@ def write_msg_parser(writer, message): msg_type = message.c_type() msg_sizeof = message.sizeof() + want_mem_size = (len(message.members) != 1 or message.members[0].is_fixed_nw_size() + or not message.members[0].is_array()) + writer.newline() parent_scope = writer.function(function_name, "uint8_t *", @@ -1014,7 +1021,9 @@ def write_msg_parser(writer, message): parent_scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); parent_scope.variable_def("uint8_t *", "start = message_start"); parent_scope.variable_def("uint8_t *", "data = NULL"); - parent_scope.variable_def("size_t", "mem_size", "nw_size"); + parent_scope.variable_def("size_t", "nw_size") + if want_mem_size: + parent_scope.variable_def("size_t", "mem_size") if not message.has_attr("nocopy"): parent_scope.variable_def("uint8_t *", "in", "end"); num_pointers = message.get_num_pointers() @@ -1026,7 +1035,8 @@ def write_msg_parser(writer, message): write_parser_helpers(writer) - write_validate_container(writer, None, message, "start", parent_scope, True, True, False) + write_validate_container(writer, None, message, "start", parent_scope, True, + want_mem_size=want_mem_size, want_extra_size=False) writer.newline() diff --git a/python_modules/marshal.py b/python_modules/marshal.py index a82df98..5b4dfdd 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -356,10 +356,12 @@ def write_message_marshaller(writer, message, is_server, private): for n in names: writer.assign("*%s_out" % n, "NULL") - src = RootMarshallingSource(None, message.c_type(), message.sizeof(), "msg") - src.reuse_scope = scope + # fix warnings about unused variables by not creating body if no members to parse + if any(x.is_fixed_nw_size() for x in message.members): + src = RootMarshallingSource(None, message.c_type(), message.sizeof(), "msg") + src.reuse_scope = scope - write_container_marshaller(writer, message, src) + write_container_marshaller(writer, message, src) writer.end_block() writer.newline() From a2257eff313f1ff06fc5f0b1192d25c75c502cd1 Mon Sep 17 00:00:00 2001 From: Uri Lublin Date: Sun, 9 Jan 2011 10:17:49 +0200 Subject: [PATCH 077/147] spice-client migration: fix minor for old migration support. For not too old spice-migration, minor is 1. For older (ancient) spice-migration, minor is 0. Affects only VM migration while a spice client is connected. --- spice1.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spice1.proto b/spice1.proto index ebb2d6f..fa2524b 100644 --- a/spice1.proto +++ b/spice1.proto @@ -168,11 +168,11 @@ channel MainChannel : BaseChannel { uint16 sport; uint32 host_offset @zero; uint32 host_size; - pubkey_type pub_key_type @minor(2); - uint32 pub_key_offset @minor(2) @zero; - uint32 pub_key_size @minor(2); + pubkey_type pub_key_type @minor(1); + uint32 pub_key_offset @minor(1) @zero; + uint32 pub_key_size @minor(1); uint8 host_data[host_size] @as_ptr @zero_terminated; - uint8 pub_key_data[pub_key_size] @minor(2) @as_ptr @zero_terminated; + uint8 pub_key_data[pub_key_size] @minor(1) @as_ptr @zero_terminated; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; From f7986c2b0d0f633f7b8953443dd4de69f37f5243 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Sat, 11 Dec 2010 15:55:19 +0200 Subject: [PATCH 078/147] python_modules/codegen.py: fix indent error in an unused function --- python_modules/codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 03c67e6..75033dc 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -11,7 +11,7 @@ def camel_to_underscores(s, upper = False): res = res + c.upper() else: res = res + c.lower() - return res + return res def underscores_to_camel(s): res = "" From fbead3281f7bf1f21f24f3087fec194787a0d030 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Sat, 9 Apr 2011 21:41:12 +0300 Subject: [PATCH 079/147] spice.proto: Fill.rop_descriptor type s/uint16/ropd (10x atiti) --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index 4eeb159..6160de1 100644 --- a/spice.proto +++ b/spice.proto @@ -639,7 +639,7 @@ channel DisplayChannel : BaseChannel { DisplayBase base; struct Fill { Brush brush @outvar(brush); - uint16 rop_descriptor; + ropd rop_descriptor; QMask mask @outvar(mask); } data; } draw_fill = 302; From 975ebc9b7d83ec6dd4fc5c256f899c8c62d35175 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Thu, 21 Apr 2011 13:16:18 +0200 Subject: [PATCH 080/147] add config.h to autogenerated files too Modify the python (de)marshaller generator to add #include at the beginning of the C files it generates --- spice_codegen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spice_codegen.py b/spice_codegen.py index 3a9989d..4dad06c 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -142,6 +142,10 @@ writer.set_option("source", os.path.basename(proto_file)) writer.public_prefix = options.prefix +writer.writeln("#ifdef HAVE_CONFIG_H") +writer.writeln("#include ") +writer.writeln("#endif") + if options.assert_on_error: writer.set_option("assert_on_error") From 12be0ad2ae2e1df18df00f4c93617076d31f45a5 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Thu, 21 Apr 2011 13:25:19 +0200 Subject: [PATCH 081/147] add comment to beginning of autogenerated files --- spice_codegen.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spice_codegen.py b/spice_codegen.py index 4dad06c..05b47f6 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -142,6 +142,8 @@ writer.set_option("source", os.path.basename(proto_file)) writer.public_prefix = options.prefix +writer.writeln("/* this is a file autogenerated by spice_codegen.py */") +writer.header.writeln("/* this is a file autogenerated by spice_codegen.py */") writer.writeln("#ifdef HAVE_CONFIG_H") writer.writeln("#include ") writer.writeln("#endif") From d14c99b84f86dc97f542df67a2fec077b05bf1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Sat, 18 Jun 2011 00:08:35 +0200 Subject: [PATCH 082/147] codegen: typedef the protocol enums Commit 9d5ef9beeca722b2ceff7d15aaa3aaaaf07ecfbf in spice-protocol introduced a typedef manually in the generated enums.h header. This patch adds them automatically to all enums during enums.h generation. --- python_modules/ptypes.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 9c4b7de..9e444f5 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -280,7 +280,7 @@ class EnumType(EnumBaseType): return "enum %s" % self.name def c_define(self, writer): - writer.write("enum ") + writer.write("typedef enum ") writer.write(self.c_name()) writer.begin_block() values = self.names.keys() @@ -296,7 +296,11 @@ class EnumType(EnumBaseType): writer.newline() writer.write(codegen.prefix_underscore_upper(self.name.upper(), "ENUM_END")) writer.newline() - writer.end_block(semicolon=True) + writer.end_block(newline=False) + writer.write(" ") + writer.write(self.c_name()) + writer.write(";") + writer.newline() writer.newline() class FlagsType(EnumBaseType): @@ -330,7 +334,7 @@ class FlagsType(EnumBaseType): return "flags %s" % self.name def c_define(self, writer): - writer.write("enum ") + writer.write("typedef enum ") writer.write(self.c_name()) writer.begin_block() values = self.names.keys() @@ -347,7 +351,11 @@ class FlagsType(EnumBaseType): writer.write(codegen.prefix_underscore_upper(self.name.upper(), "MASK")) writer.write(" = 0x%x" % (mask)) writer.newline() - writer.end_block(semicolon=True) + writer.end_block(newline=False) + writer.write(" ") + writer.write(self.c_name()) + writer.write(";") + writer.newline() writer.newline() class ArrayType(Type): From bbd93cdb6b26e475bc7772a6b0c239082adfc153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 21 Jun 2011 13:20:33 +0200 Subject: [PATCH 083/147] python: remove c-ism trailing ; --- python_modules/codegen.py | 6 +++--- python_modules/demarshal.py | 32 ++++++++++++++++---------------- python_modules/ptypes.py | 6 +++--- python_modules/spice_parser.py | 2 +- spice_codegen.py | 4 ++-- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 75033dc..116760c 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -116,7 +116,7 @@ class CodeWriter: writer.options = self.options writer.public_prefix = self.public_prefix - return writer; + return writer def write(self, s): # Ensure its a string @@ -184,10 +184,10 @@ class CodeWriter: self.statement("goto %s" % label) def indent(self): - self.indentation += 4; + self.indentation += 4 def unindent(self): - self.indentation -= 4; + self.indentation -= 4 if self.indentation < 0: self.indenttation = 0 diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 9d3b1e4..3a0178e 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -74,7 +74,7 @@ def write_parser_helpers(writer): writer.newline() writer.statement("typedef struct PointerInfo PointerInfo") - writer.statement("typedef void (*message_destructor_t)(uint8_t *message)"); + writer.statement("typedef void (*message_destructor_t)(uint8_t *message)") writer.statement("typedef uint8_t * (*parse_func_t)(uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *ptr_info, int minor)") writer.statement("typedef uint8_t * (*parse_msg_func_t)(uint8_t *message_start, uint8_t *message_end, int minor, size_t *size_out, message_destructor_t *free_message)") writer.statement("typedef uint8_t * (*spice_parse_channel_func_t)(uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message)") @@ -198,11 +198,11 @@ def write_validate_struct_function(writer, struct): writer = writer.function_helper() scope = writer.function(validate_function, "static intptr_t", "uint8_t *message_start, uint8_t *message_end, uint64_t offset, int minor") scope.variable_def("uint8_t *", "start = message_start + offset") - scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); - scope.variable_def("size_t", "mem_size", "nw_size"); + scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos") + scope.variable_def("size_t", "mem_size", "nw_size") num_pointers = struct.get_num_pointers() if num_pointers != 0: - scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size"); + scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size") writer.newline() with writer.if_block("offset == 0"): @@ -766,8 +766,8 @@ def write_parse_ptr_function(writer, target_type): num_pointers = target_type.get_num_pointers() if num_pointers != 0: - scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size"); - scope.variable_def("uint32_t", "n_ptr=0"); + scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size") + scope.variable_def("uint32_t", "n_ptr=0") scope.variable_def("PointerInfo", "ptr_info[%s]" % num_pointers) writer.newline() @@ -854,7 +854,7 @@ def write_parse_pointer_core(writer, target_type, offset, at_end, dest, member_n writer.assign("ptr_info[n_ptr].parse", write_parse_ptr_function(writer, target_type)) if at_end: writer.assign("ptr_info[n_ptr].dest", "(void **)end") - writer.increment("end", "sizeof(void *)"); + writer.increment("end", "sizeof(void *)") else: writer.assign("ptr_info[n_ptr].dest", "(void **)&%s" % dest.get_ref(member_name)) if target_type.is_array(): @@ -890,7 +890,7 @@ def write_member_parser(writer, container, member, dest, scope): assert(t.target_type.is_array()) nelements = read_array_len(writer, member.name, t.target_type, dest, scope, True) writer.comment("Reuse data from network message as chunk").newline() - scope.variable_def("SpiceChunks *", "chunks"); + scope.variable_def("SpiceChunks *", "chunks") writer.assign("chunks", "(SpiceChunks *)end") writer.increment("end", "sizeof(SpiceChunks) + sizeof(SpiceChunk)") writer.assign(dest.get_ref(member.name), "chunks") @@ -922,7 +922,7 @@ def write_member_parser(writer, container, member, dest, scope): if member.has_attr("chunk") and t.element_type.is_fixed_nw_size() and t.element_type.get_fixed_nw_size() == 1: writer.comment("use array as chunk").newline() - scope.variable_def("SpiceChunks *", "chunks"); + scope.variable_def("SpiceChunks *", "chunks") writer.assign("chunks", "(SpiceChunks *)end") writer.increment("end", "sizeof(SpiceChunks) + sizeof(SpiceChunk)") writer.assign(dest.get_ref(member.name), "chunks") @@ -1018,18 +1018,18 @@ def write_msg_parser(writer, message): parent_scope = writer.function(function_name, "uint8_t *", "uint8_t *message_start, uint8_t *message_end, int minor, size_t *size, message_destructor_t *free_message", True) - parent_scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos"); - parent_scope.variable_def("uint8_t *", "start = message_start"); - parent_scope.variable_def("uint8_t *", "data = NULL"); + parent_scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos") + parent_scope.variable_def("uint8_t *", "start = message_start") + parent_scope.variable_def("uint8_t *", "data = NULL") parent_scope.variable_def("size_t", "nw_size") if want_mem_size: parent_scope.variable_def("size_t", "mem_size") if not message.has_attr("nocopy"): - parent_scope.variable_def("uint8_t *", "in", "end"); + parent_scope.variable_def("uint8_t *", "in", "end") num_pointers = message.get_num_pointers() if num_pointers != 0: - parent_scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size"); - parent_scope.variable_def("uint32_t", "n_ptr=0"); + parent_scope.variable_def("SPICE_GNUC_UNUSED intptr_t", "ptr_size") + parent_scope.variable_def("uint32_t", "n_ptr=0") parent_scope.variable_def("PointerInfo", "ptr_info[%s]" % num_pointers) writer.newline() @@ -1121,7 +1121,7 @@ def write_channel_parser(writer, channel, server): d = 0 for r in ranges: d = d + 1 - writer.write("static parse_msg_func_t funcs%d[%d] = " % (d, r[1] - r[0])); + writer.write("static parse_msg_func_t funcs%d[%d] = " % (d, r[1] - r[0])) writer.begin_block() for i in range(r[0], r[1]): func = write_msg_parser(helpers, ids[i].message_type) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 9e444f5..f12a2f3 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -37,10 +37,10 @@ class FixedSize: new.vals[i] = self.vals[i] + other.vals[i] for i in range(shared,len(self.vals)): - new.vals[i] = self.vals[i]; + new.vals[i] = self.vals[i] for i in range(shared,len(other.vals)): - new.vals[i] = new.vals[i] + other.vals[i]; + new.vals[i] = new.vals[i] + other.vals[i] return new @@ -689,7 +689,7 @@ class Switch(Containee): def get_fixed_nw_size(self): if not self.is_fixed_nw_size(): raise Exception, "Not a fixed size type" - size = 0; + size = 0 for c in self.cases: size = max(size, c.member.get_fixed_nw_size()) return size diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index 43e930c..4c8a57a 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -18,7 +18,7 @@ def parseVariableDef(toks): t = ptypes.ArrayType(t, array_size) if pointer != None: - t = ptypes.PointerType(t); + t = ptypes.PointerType(t) return ptypes.Member(name, t, attributes) diff --git a/spice_codegen.py b/spice_codegen.py index 05b47f6..c6d6aa9 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -16,7 +16,7 @@ def write_channel_enums(writer, channel, client): if len(messages) == 0: return writer.begin_block("enum") - i = 0; + i = 0 if client: prefix = [ "MSGC" ] else: @@ -53,7 +53,7 @@ def write_enums(writer): if isinstance(t, ptypes.EnumBaseType): t.c_define(writer) - i = 0; + i = 0 writer.begin_block("enum") for c in proto.channels: enum = codegen.prefix_underscore_upper("CHANNEL", c.name.upper()) From da5cad88e37f8d3968cb8639e91ca803f8a84d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Mon, 23 May 2011 12:20:17 +0200 Subject: [PATCH 084/147] sndworker: add AudioVolume/AudioMute messages These messages allow the guest to send the audio device volume to the client. It uses an arbitrary scale of 16bits, which works good enough for now. Save VolumeState in {Playback,Record}State, so that we can send the current volume on channel connection. Note about future improvements: - add exact dB support - add client to guest volume change Updated since v2: - bumped record and playback interface minor version to allow conditional compilation Updated since v1: - sync record volume on connection too --- spice.proto | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/spice.proto b/spice.proto index 6160de1..80c40d4 100644 --- a/spice.proto +++ b/spice.proto @@ -926,6 +926,15 @@ enum16 audio_fmt { S16, }; +message AudioVolume { + uint8 nchannels; + uint16 volume[nchannels] @end; +}; + +message AudioMute { + uint8 mute; +}; + channel PlaybackChannel : BaseChannel { server: message { @@ -947,6 +956,8 @@ channel PlaybackChannel : BaseChannel { } start; Empty stop; + AudioVolume volume; + AudioMute mute; }; channel RecordChannel : BaseChannel { @@ -958,6 +969,8 @@ channel RecordChannel : BaseChannel { } start = 101; Empty stop; + AudioVolume volume; + AudioMute mute; client: message { uint32 time; From a11615f71391c1ef7e578eaec1c57fcd0f076420 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Thu, 19 May 2011 15:59:07 +0200 Subject: [PATCH 085/147] add check for pyparsing Check both in configure.ac (after checking if we need to rebuild the marshalling files) and in the python script using pyparsing (for people modifying .proto files in tarballs) --- python_modules/spice_parser.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index 4c8a57a..e20e8fc 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -1,6 +1,11 @@ -from pyparsing import Literal, CaselessLiteral, Word, OneOrMore, ZeroOrMore, \ - Forward, delimitedList, Group, Optional, Combine, alphas, nums, restOfLine, cStyleComment, \ - alphanums, ParseException, ParseResults, Keyword, StringEnd, replaceWith +try: + from pyparsing import Literal, CaselessLiteral, Word, OneOrMore, ZeroOrMore, \ + Forward, delimitedList, Group, Optional, Combine, alphas, nums, restOfLine, cStyleComment, \ + alphanums, ParseException, ParseResults, Keyword, StringEnd, replaceWith +except ImportError: + print "Module pyparsing not found." + exit(1) + import ptypes import sys From 4b6598dc0286c9b9d5ad40b7046172b4d118ba3c Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 22 Jun 2011 10:58:57 +0200 Subject: [PATCH 086/147] don't #include config.h in generated header files 7e30572ab adds a #include to the beginning of generated files. It also does this for generated headers and enums files, which is not wanted, especially if it's an installed file. This commit only adds this include for the non-header non-enum case (hopefully, enums are only generated for use in a .h file). --- spice_codegen.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spice_codegen.py b/spice_codegen.py index c6d6aa9..a9fdee9 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -144,9 +144,10 @@ writer.public_prefix = options.prefix writer.writeln("/* this is a file autogenerated by spice_codegen.py */") writer.header.writeln("/* this is a file autogenerated by spice_codegen.py */") -writer.writeln("#ifdef HAVE_CONFIG_H") -writer.writeln("#include ") -writer.writeln("#endif") +if not options.header and not options.generate_enums: + writer.writeln("#ifdef HAVE_CONFIG_H") + writer.writeln("#include ") + writer.writeln("#endif") if options.assert_on_error: writer.set_option("assert_on_error") From 85da46bacd4c54559ed8c6ca0386b1da5c1fe65f Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 25 Jul 2011 11:09:12 +0200 Subject: [PATCH 087/147] server: Add a usbredir channel --- spice.proto | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spice.proto b/spice.proto index 80c40d4..99302ff 100644 --- a/spice.proto +++ b/spice.proto @@ -1104,6 +1104,13 @@ client: Data data = 101; }; +channel UsbredirChannel : BaseChannel { +server: + Data data = 101; +client: + Data data = 101; +}; + protocol Spice { MainChannel main = 1; DisplayChannel display; @@ -1113,4 +1120,5 @@ protocol Spice { RecordChannel record; TunnelChannel tunnel; SmartcardChannel smartcard; + UsbredirChannel usbredir; }; From 40ef2d54596a59f64f1d21f9f009f2361892162b Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Sat, 2 Jul 2011 04:23:45 +0200 Subject: [PATCH 088/147] spice.proto: add comment for origin of STYLED and START_WITH_GAP --- spice.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spice.proto b/spice.proto index 99302ff..abf3ec3 100644 --- a/spice.proto +++ b/spice.proto @@ -339,6 +339,11 @@ flags16 ropd { INVERS_RES, }; +/* This *must* remain with values identical to api/winddi.h + LA_STYLED == 0x8 (log_2)=> 3 + LA_STARTGAP == 0x4 (log_2)=> 2 + This is used by the windows driver. + */ flags8 line_flags { STYLED = 3, START_WITH_GAP = 2, From b530ce7ea22fd5de6a53ef915fc787f0b3e67c05 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Sun, 18 Sep 2011 10:31:38 +0300 Subject: [PATCH 089/147] server,proto: tell the clients to connect to the migration target before migraton starts (1) send SPICE_MSG_MAIN_MIGRATE_BEGIN upon spice_server_migrate_connect (to all the clients that support it) (2) wait for SPICE_MSGC_MAIN_MIGRATE_(CONNECTED|CONNECT_ERROR) from all the relevant clients, or a timeout, in order to complete client_migrate_info monitor command (cherry picked from commit 5560c56ef05c74da5e0e0825dc1f134019593cad branch 0.8; Was modified to support the separation of main channel from reds, and multiple clients) Conflicts: server/reds.c --- spice.proto | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spice.proto b/spice.proto index abf3ec3..78c1fad 100644 --- a/spice.proto +++ b/spice.proto @@ -167,9 +167,8 @@ channel MainChannel : BaseChannel { uint16 sport; uint32 host_size; uint8 *host_data[host_size] @zero_terminated @marshall @nonnull; - pubkey_type pub_key_type; - uint32 pub_key_size; - uint8 *pub_key_data[pub_key_size] @zero_terminated @marshall @nonnull; + uint32 cert_subject_size; + uint8 *cert_subject_data[cert_subject_size] @zero_terminated @marshall; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; From f74d9176a72ea758fc35ea34803c7041cf3e3ddc Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Sun, 18 Sep 2011 11:10:44 +0300 Subject: [PATCH 090/147] spice.proto: add SPICE_MSG_MAIN_MIGRATE_END & SPICE_MSGC_MAIN_MIGRATE_END (cherry picked from commit cfbd07710562e522179ae5a7085a789489a821bb branch 0.8) --- spice.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spice.proto b/spice.proto index 78c1fad..266e03a 100644 --- a/spice.proto +++ b/spice.proto @@ -219,6 +219,8 @@ channel MainChannel : BaseChannel { uint8 *cert_subject_data[cert_subject_size] @zero_terminated @marshall; } @ctype(SpiceMsgMainMigrationSwitchHost) migrate_switch_host; + Empty migrate_end; + client: message { uint64 cache_size; @@ -243,6 +245,8 @@ channel MainChannel : BaseChannel { message { uint32 num_tokens; } @ctype(SpiceMsgcMainAgentTokens) agent_token; + + Empty migrate_end; }; enum8 clip_type { From cdb54a8fa557edb5a61119d372722028389e6143 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Thu, 12 Jan 2012 13:24:52 +0100 Subject: [PATCH 091/147] codegen: Fix enums.h generation With the new usbredir code we have the new concept of the abstract / generic spicevmc channel type (which just tunnels data from a qemu chardev), and we've the usbredir channel, which is the only current user of this. This was reflected in the protocols enum in spice-protocol.h by a manual edit done by me, my bad. This patch teaches spice.proto about the relation between the abstract spicevmc channel and the usbredir channel and modifies codegen to deal with this. Signed-off-by: Hans de Goede --- python_modules/ptypes.py | 4 +++- spice.proto | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index f12a2f3..4585949 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -947,6 +947,9 @@ class ChannelType(Type): server_messages_byname = self.base.server_messages_byname.copy() client_messages = self.base.client_messages[:] client_messages_byname = self.base.client_messages_byname.copy() + + # Set default member_name, FooChannel -> foo + self.member_name = self.name[:-7].lower() else: server_messages = [] server_messages_byname = {} @@ -998,7 +1001,6 @@ class ProtocolMember: def resolve(self, protocol): self.channel_type = self.channel_type.resolve() - assert(self.channel_type.member_name == None) self.channel_type.member_name = self.name return self diff --git a/spice.proto b/spice.proto index 266e03a..4d0731b 100644 --- a/spice.proto +++ b/spice.proto @@ -1112,13 +1112,16 @@ client: Data data = 101; }; -channel UsbredirChannel : BaseChannel { +channel SpicevmcChannel : BaseChannel { server: Data data = 101; client: Data data = 101; }; +channel UsbredirChannel : SpicevmcChannel { +}; + protocol Spice { MainChannel main = 1; DisplayChannel display; From b444b766551a7506444d3b8230316ed0f4e6b199 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Tue, 10 Jan 2012 09:29:27 +0200 Subject: [PATCH 092/147] spice.proto: add SPICE_MSG_LIST to base channel --- spice.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/spice.proto b/spice.proto index 4d0731b..0e15fe7 100644 --- a/spice.proto +++ b/spice.proto @@ -133,6 +133,7 @@ channel BaseChannel { uint8 message[message_len] @end @nomarshal; } notify; + Data list; /* the msg body is SpiceSubMessageList */ client: message { uint32 generation; From ce59cc14167979a54f5846a59d58db132f4c6048 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrange" Date: Tue, 10 Jan 2012 15:03:38 +0000 Subject: [PATCH 093/147] Remove trailing blank lines Remove any blank lines at the end of all source files --- python_modules/Makefile.am | 1 - python_modules/spice_parser.py | 1 - 2 files changed, 2 deletions(-) diff --git a/python_modules/Makefile.am b/python_modules/Makefile.am index f304ec0..7d416df 100644 --- a/python_modules/Makefile.am +++ b/python_modules/Makefile.am @@ -3,4 +3,3 @@ NULL = PYTHON_MODULES = __init__.py codegen.py demarshal.py marshal.py ptypes.py spice_parser.py EXTRA_DIST = $(PYTHON_MODULES) - diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index e20e8fc..d0aabb0 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -159,4 +159,3 @@ def parse(filename): t.register() protocol = types[-1] return protocol - From 36ea49b68619a0d35c2887a1c7a3960dbc4d4fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Fri, 2 Mar 2012 13:42:10 +0100 Subject: [PATCH 094/147] demarshal: fixed-size array are missing __nelements A message with a fixed-size array, such as uint8 uuid[16] will generate an invalid code, missing the __nelements variable. Make sure that variable is defined. --- python_modules/demarshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 3a0178e..541735a 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -664,7 +664,7 @@ def read_array_len(writer, prefix, array, dest, scope, is_ptr): nelements = "%s__array__nelements" % prefix else: nelements = "%s__nelements" % prefix - if dest.is_toplevel(): + if dest.is_toplevel() and scope.variable_defined(nelements): return nelements # Already there for toplevel, need not recalculate element_type = array.element_type scope.variable_def("uint32_t", nelements) From 7220080972f93700f3620497a2ca69b96d587cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Fri, 2 Mar 2012 13:45:15 +0100 Subject: [PATCH 095/147] Send name & uuid to capable clients Add spice_server_set_name() and spice_server_set_uuid() that allows the client to identify a Spice server (useful to associate settings with a particular server) The SPICE_MSG_MAIN_NAME and SPICE_MSG_MAIN_UUID messages are only sent to capable clients, announcing SPICE_MAIN_CAP_NAME_AND_UUID. --- spice.proto | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spice.proto b/spice.proto index 0e15fe7..ae27c8d 100644 --- a/spice.proto +++ b/spice.proto @@ -134,6 +134,7 @@ channel BaseChannel { } notify; Data list; /* the msg body is SpiceSubMessageList */ + client: message { uint32 generation; @@ -222,6 +223,15 @@ channel MainChannel : BaseChannel { Empty migrate_end; + message { + uint32 name_len; + uint8 name[name_len]; + } name; + + message { + uint8 uuid[16]; + } uuid; + client: message { uint64 cache_size; From 7f23ab54aae7dc820380d54ad99decc176e93ff1 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Thu, 5 May 2011 17:57:33 +0200 Subject: [PATCH 096/147] fix copy & paste error in ptypes.py --- python_modules/ptypes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 4585949..7b30311 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -234,9 +234,7 @@ class EnumBaseType(Type): return codegen.prefix_camel(self.name) def c_enumname(self, value): - if self.has_attr("prefix"): - return self.attributes["prefix"][0] + self.names[value] - return codegen.prefix_underscore_upper(self.name.upper(), self.names[value]) + self.c_enumname_by_name(self.names[value]) def c_enumname_by_name(self, name): if self.has_attr("prefix"): From d8975877c6e275961ee7d891c8dd2fc67d30305e Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 22 Jun 2011 13:22:22 +0200 Subject: [PATCH 097/147] add ifdef/endif methods to spice code generator These methods will be needed to be able to make some fields optional in spice.proto --- python_modules/codegen.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 116760c..6d53551 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -322,6 +322,18 @@ class CodeWriter: def macro(self, name, args, define): self.write("#define %s(%s) %s" % (name, args, define)).newline() + def ifdef(self, name): + indentation = self.indentation + self.indentation = 0; + self.write("#ifdef %s" % (name)).newline() + self.indentation = indentation + + def endif(self, name): + indentation = self.indentation + self.indentation = 0; + self.write("#endif /* %s */" % (name)).newline() + self.indentation = indentation + def add_function_variable(self, ctype, name): if self.function_variables.has_key(name): assert(self.function_variables[name] == ctype) From 0067c31bad1fadbfceba7014497374b9daf9a6a0 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 22 Jun 2011 13:23:47 +0200 Subject: [PATCH 098/147] allow attributes on channel elements in .proto files We want to be able to add an @ifdef annotation to optional messages For example, we want to compile in the smartcard messages only if libcacard is available --- python_modules/ptypes.py | 4 +++- python_modules/spice_parser.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 7b30311..132ba99 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -915,12 +915,14 @@ class ChannelMember(Containee): return "%s (%s)" % (str(self.name), str(self.message_type)) class ChannelType(Type): - def __init__(self, name, base, members): + def __init__(self, name, base, members, attribute_list): Type.__init__(self) self.name = name self.base = base self.member_name = None self.members = members + for attr in attribute_list: + self.attributes[attr[0][1:]] = attr[1:] def __str__(self): if self.name == None: diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index d0aabb0..44456ab 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -124,7 +124,7 @@ def SPICE_BNF(): enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction(lambda toks: ptypes.EnumType(toks[0][0], toks[0][1], toks[0][2], toks[0][3])) flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction(lambda toks: ptypes.FlagsType(toks[0][0], toks[0][1], toks[0][2], toks[0][3])) messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction(lambda toks: ptypes.MessageType(toks[0][1], toks[0][2], toks[0][3])) - channelDef = Group(channel_ + identifier + channelBody - semi).setParseAction(lambda toks: ptypes.ChannelType(toks[0][1], toks[0][2], toks[0][3])) + channelDef = Group(channel_ + identifier + channelBody + attributes - semi).setParseAction(lambda toks: ptypes.ChannelType(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction(lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3])) typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction(lambda toks: ptypes.TypeAlias(toks[0][1], toks[0][2], toks[0][3])) From 12a5ce3dc5d9b130de1eba7038c9915428fac802 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 22 Jun 2011 13:26:15 +0200 Subject: [PATCH 099/147] handle @ifdef on messages and channels --- python_modules/demarshal.py | 22 ++++++++++++++++++---- python_modules/marshal.py | 28 +++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 541735a..2381e7f 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1015,6 +1015,8 @@ def write_msg_parser(writer, message): or not message.members[0].is_array()) writer.newline() + if message.has_attr("ifdef"): + writer.ifdef(message.attributes["ifdef"][0]) parent_scope = writer.function(function_name, "uint8_t *", "uint8_t *message_start, uint8_t *message_end, int minor, size_t *size, message_destructor_t *free_message", True) @@ -1084,6 +1086,9 @@ def write_msg_parser(writer, message): writer.statement("return NULL") writer.end_block() + if message.has_attr("ifdef"): + writer.endif(message.attributes["ifdef"][0]) + return function_name def write_channel_parser(writer, channel, server): @@ -1112,6 +1117,8 @@ def write_channel_parser(writer, channel, server): else: function_name = "parse_%s_msgc" % channel.name writer.newline() + if channel.has_attr("ifdef"): + writer.ifdef(channel.attributes["ifdef"][0]) scope = writer.function(function_name, "static uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") @@ -1141,6 +1148,8 @@ def write_channel_parser(writer, channel, server): writer.statement("return NULL") writer.end_block() + if channel.has_attr("ifdef"): + writer.endif(channel.attributes["ifdef"][0]) return function_name @@ -1157,13 +1166,16 @@ def write_get_channel_parser(writer, channel_parsers, max_channel, is_server): writer.write("static struct {spice_parse_channel_func_t func; unsigned int max_messages; } channels[%d] = " % (max_channel+1)) writer.begin_block() + channel = None for i in range(0, max_channel + 1): - writer.write("{ ") if channel_parsers.has_key(i): + channel = channel_parsers[i][0] + if channel.has_attr("ifdef"): + writer.ifdef(channel.attributes["ifdef"][0]) + writer.write("{ ") writer.write(channel_parsers[i][1]) writer.write(", ") - channel = channel_parsers[i][0] max_msg = 0 if is_server: messages = channel.server_messages @@ -1172,13 +1184,15 @@ def write_get_channel_parser(writer, channel_parsers, max_channel, is_server): for m in messages: max_msg = max(max_msg, m.value) writer.write(max_msg) + writer.write("}") else: - writer.write("NULL, 0") - writer.write("}") + writer.write("{ NULL, 0 }") if i != max_channel: writer.write(",") writer.newline() + if channel and channel.has_attr("ifdef"): + writer.endif(channel.attributes["ifdef"][0]) writer.end_block(semicolon = True) with writer.if_block("channel < %d" % (max_channel + 1)): diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 5b4dfdd..d93f983 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -333,6 +333,8 @@ def write_container_marshaller(writer, container, src): write_member_marshaller(writer, container, m, src, scope) def write_message_marshaller(writer, message, is_server, private): + if message.has_attr("ifdef"): + writer.ifdef(message.attributes["ifdef"][0]) writer.out_prefix = "" function_name = "spice_marshall_" + message.c_name() if writer.is_generated("marshaller", function_name): @@ -364,6 +366,8 @@ def write_message_marshaller(writer, message, is_server, private): write_container_marshaller(writer, message, src) writer.end_block() + if message.has_attr("ifdef"): + writer.endif(message.attributes["ifdef"][0]) writer.newline() return function_name @@ -371,16 +375,30 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): functions = {} for c in proto.channels: channel = c.channel_type + if channel.has_attr("ifdef"): + writer.ifdef(channel.attributes["ifdef"][0]) if is_server: for m in channel.client_messages: message = m.message_type f = write_message_marshaller(writer, message, is_server, private_marshallers) - functions[f] = True + if channel.has_attr("ifdef") and not functions.has_key(f): + functions[f] = channel.attributes["ifdef"][0] + elif message.has_attr("ifdef") and not functions.has_key(f): + functions[f] = message.attributes["ifdef"][0] + else: + functions[f] = True else: for m in channel.server_messages: message = m.message_type - f= write_message_marshaller(writer, message, is_server, private_marshallers) - functions[f] = True + f = write_message_marshaller(writer, message, is_server, private_marshallers) + if channel.has_attr("ifdef") and not functions.has_key(f): + functions[f] = channel.attributes["ifdef"][0] + elif message.has_attr("ifdef") and not functions.has_key(f): + functions[f] = message.attributes["ifdef"][0] + else: + functions[f] = True + if channel.has_attr("ifdef"): + writer.endif(channel.attributes["ifdef"][0]) if private_marshallers: scope = writer.function("spice_message_marshallers_get" + writer.public_prefix, @@ -391,7 +409,11 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): member = f[len("spice_marshall_"):] if not member.startswith("msg"): member = "msg_" + member + if functions[f] != True: + writer.ifdef(functions[f]) writer.assign("marshallers.%s" % member, f) + if functions[f] != True: + writer.endif(functions[f]) writer.newline() writer.statement("return &marshallers") From ca57afda341a1187a9c367e5be71744979dd7a86 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Fri, 30 Sep 2011 11:19:38 +0200 Subject: [PATCH 100/147] spice_codegen: Always write a channels entry for an ifdef-ed channel Before this patch, if a channel is defined conditionally in spice.proto (because it depends on external headers like the smartcard channel), spice_codegen would write an entry to the channels array in spice_get_*_channel_parser which would only take up a place in the array if the ifdef condition is true, thus moving up all other intializers one place when it is not true. This was causing issues (crashes) when building spice-gtk with the combination of usbredir support enabled and smartcard support disabled. This patch fixes this by adding #else { NULL, 0 }, to the generated code. Thanks to coolper chen for reporting this! Signed-off-by: Hans de Goede --- python_modules/codegen.py | 6 ++++++ python_modules/demarshal.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 6d53551..009cf95 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -328,6 +328,12 @@ class CodeWriter: self.write("#ifdef %s" % (name)).newline() self.indentation = indentation + def ifdef_else(self, name): + indentation = self.indentation + self.indentation = 0; + self.write("#else /* %s */" % (name)).newline() + self.indentation = indentation + def endif(self, name): indentation = self.indentation self.indentation = 0; diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 2381e7f..c8c6837 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1192,6 +1192,11 @@ def write_get_channel_parser(writer, channel_parsers, max_channel, is_server): writer.write(",") writer.newline() if channel and channel.has_attr("ifdef"): + writer.ifdef_else(channel.attributes["ifdef"][0]) + writer.write("{ NULL, 0 }") + if i != max_channel: + writer.write(",") + writer.newline() writer.endif(channel.attributes["ifdef"][0]) writer.end_block(semicolon = True) From 617b0adac30e01b81709dc6047f42fc8e0ea915e Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Thu, 5 May 2011 16:39:23 +0200 Subject: [PATCH 101/147] add smartcard bits to spice.proto --- spice.proto | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/spice.proto b/spice.proto index ae27c8d..de238c7 100644 --- a/spice.proto +++ b/spice.proto @@ -1116,11 +1116,80 @@ channel TunnelChannel : BaseChannel { } @ctype(SpiceMsgcTunnelSocketTokens) socket_token; }; +enum32 vsc_message_type { + Init = 1, + Error, + ReaderAdd, + ReaderRemove, + ATR, + CardRemove, + APDU, + Flush, + FlushComplete +} @prefix(VSC_); + +struct VscMessageHeader { + vsc_message_type type; + uint32 reader_id; + uint32 length; +} @ctype(VSCMsgHeader); + +struct VscMessageError { + uint32 code; +} @ctype(VSCMsgError); + +struct VscMessageAPDU { + uint8 data[]; +} @ctype(VSCMsgAPDU); + +struct VscMessageATR { + uint8 data[]; +} @ctype(VSCMsgATR); + +struct VscMessageReaderAdd { + int8 *reader_name[] @zero_terminated @nonnull @end @nomarshal; +} @ctype(VSCMsgReaderAdd); + channel SmartcardChannel : BaseChannel { -server: - Data data = 101; -client: - Data data = 101; + server: + message { + vsc_message_type type; + uint32 reader_id; + uint32 length; + uint8 data[] @end; + } @ctype(SpiceMsgSmartcard) msg = 101; + + client: + message { + VscMessageHeader header; + switch (header.type) { + case ReaderAdd: + VscMessageReaderAdd add; + case ATR: + case APDU: + VscMessageATR atr_data; + case Error: + VscMessageError error; + } u @anon; + } @ctype(SpiceMsgcSmartcard) msg = 101; + + message { + vsc_message_type type; + uint32 reader_id; + uint32 length; + } @ctype(VSCMsgHeader) header = 101; + + message { + uint32 code; + } @ctype(VSCMsgError) error = 101; + + message { + uint8 data[]; + } @ctype(VSCMsgATR) atr = 101; + + message { + int8 reader_name[] @zero_terminated @nonnull; + } @ctype(VSCMsgReaderAdd) reader_add = 101; }; channel SpicevmcChannel : BaseChannel { From 94a46ebc2a3d83802c8ab297b4c2f4441def0ee2 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 22 Jun 2011 13:54:55 +0200 Subject: [PATCH 102/147] use new @ifdef directive for smartcard messages We don't want to conditionally compile the smartcard messages depending on whether USE_SMARTCARD is set or not, we can now use the @ifdef attribute for that. --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index de238c7..e3feffa 100644 --- a/spice.proto +++ b/spice.proto @@ -1190,7 +1190,7 @@ channel SmartcardChannel : BaseChannel { message { int8 reader_name[] @zero_terminated @nonnull; } @ctype(VSCMsgReaderAdd) reader_add = 101; -}; +} @ifdef(USE_SMARTCARD); channel SpicevmcChannel : BaseChannel { server: From ecce70452f5e28e111fb7acd6eac5c169ba67cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 21 Mar 2012 01:43:29 +0100 Subject: [PATCH 103/147] codegen: struct marshallers are not current function helper This solves the issue of struct_marshallers being included within the current ifdef/endif body, although they are independant functions. --- python_modules/marshal.py | 5 +++-- spice_codegen.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index d93f983..dc10a4c 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -99,7 +99,7 @@ class SubMarshallingSource(MarshallingSource): else: return self.parent_src.get_ref(self.name) + "." + member -def write_marshal_ptr_function(writer, target_type): +def write_marshal_ptr_function(writer, target_type, is_helper=True): if target_type.is_array(): marshal_function = "spice_marshall_array_%s" % target_type.element_type.primitive_type() else: @@ -116,7 +116,8 @@ def write_marshal_ptr_function(writer, target_type): names_args = "".join(n) header = writer.header - writer = writer.function_helper() + if is_helper: + writer = writer.function_helper() writer.header = header writer.out_prefix = "" if target_type.is_array(): diff --git a/spice_codegen.py b/spice_codegen.py index a9fdee9..759b09d 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -188,7 +188,7 @@ if options.generate_marshallers: if options.struct_marshallers: for structname in options.struct_marshallers: t = ptypes.lookup_type(structname) - marshal.write_marshal_ptr_function(writer, t) + marshal.write_marshal_ptr_function(writer, t, False) if options.generate_marshallers or (options.struct_marshallers and len(options.struct_marshallers) > 0): marshal.write_trailer(writer) From 055c68ccdd33cb9d22ee34275aceb345092955f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 21 Mar 2012 01:45:04 +0100 Subject: [PATCH 104/147] codegen: include headers locally --- python_modules/marshal.py | 4 ++-- spice_codegen.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index dc10a4c..8cbc426 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -4,7 +4,7 @@ import codegen def write_includes(writer): writer.header.writeln("#include ") - writer.header.writeln("#include ") + writer.header.writeln('#include "marshaller.h"') writer.header.newline() writer.header.writeln("#ifndef _GENERATED_HEADERS_H") writer.header.writeln("#define _GENERATED_HEADERS_H") @@ -15,7 +15,7 @@ def write_includes(writer): writer.writeln("#include ") writer.writeln("#include ") writer.writeln("#include ") - writer.writeln("#include ") + writer.writeln('#include "marshaller.h"') writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") diff --git a/spice_codegen.py b/spice_codegen.py index 759b09d..e9e64c0 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -157,6 +157,7 @@ if options.print_error: if options.includes: for i in options.includes: + writer.header.writeln('#include "%s"' % i) writer.writeln('#include "%s"' % i) if options.generate_enums: From 3215ddd6cd73adef6d0a7774ece46c3526b0b26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 21 Mar 2012 01:46:44 +0100 Subject: [PATCH 105/147] codegen: ifdef/endif function declaration too Compile out part that we are not supporting. In the future, we might want to declare a fake type and an empty function to keep API compatibility --- python_modules/marshal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 8cbc426..4020799 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -378,6 +378,7 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): channel = c.channel_type if channel.has_attr("ifdef"): writer.ifdef(channel.attributes["ifdef"][0]) + writer.header.ifdef(channel.attributes["ifdef"][0]) if is_server: for m in channel.client_messages: message = m.message_type @@ -400,6 +401,7 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): functions[f] = True if channel.has_attr("ifdef"): writer.endif(channel.attributes["ifdef"][0]) + writer.header.endif(channel.attributes["ifdef"][0]) if private_marshallers: scope = writer.function("spice_message_marshallers_get" + writer.public_prefix, From 15dba2901b8927f024c6f496a468557c8ea64119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 20 Mar 2012 21:38:33 +0100 Subject: [PATCH 106/147] spice.proto: fix demarshaller crash with name message It turned out the demarshaller wasn't allocating enough space to memcpy the name. In order to take into account the size of a variable array, it needs to be marked with the @end tag so that the "extra_size" is added to the allocated memory. It would be nice if the demarshaller would somehow fail if this wasn't set explicitly, or do the right thing by default. @end the name so that demarshaller --- spice.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index e3feffa..513fe87 100644 --- a/spice.proto +++ b/spice.proto @@ -225,7 +225,7 @@ channel MainChannel : BaseChannel { message { uint32 name_len; - uint8 name[name_len]; + uint8 name[name_len] @end; } name; message { From 534d35cc328dd8d2002d0b0efc4d281b82e8e1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 20 Mar 2012 12:53:30 +0100 Subject: [PATCH 107/147] build-sys: make it a seperately buildable spice-common library - autotoolize - fix headers inclusion - generate gitignores - workaround serverSMARTCARD support with dirty hack... --- python_modules/Makefile.am | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python_modules/Makefile.am b/python_modules/Makefile.am index 7d416df..50e1a71 100644 --- a/python_modules/Makefile.am +++ b/python_modules/Makefile.am @@ -1,5 +1,16 @@ NULL = -PYTHON_MODULES = __init__.py codegen.py demarshal.py marshal.py ptypes.py spice_parser.py +PYTHON_MODULES = \ + __init__.py \ + codegen.py \ + demarshal.py \ + marshal.py \ + ptypes.py \ + spice_parser.py \ + $(NULL) EXTRA_DIST = $(PYTHON_MODULES) + +DISTCLEANFILES = *.pyc + +-include $(top_srcdir)/git.mk From 725b5a9b53276aad505501e04f3d1cfbd7b6822f Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Fri, 6 Apr 2012 15:25:34 +0200 Subject: [PATCH 108/147] demarshall: add missing parens in BE read_xxx functions The missing parens causes build to fail on big-endian machines --- python_modules/demarshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index c8c6837..cf48d74 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -50,7 +50,7 @@ def write_parser_helpers(writer): writer.macro("read_%s" % type, "ptr", "(*((%s_t *)(ptr)))" % type) writer.macro("write_%s" % type, "ptr, val", "*(%s_t *)(ptr) = val" % (type)) else: - writer.macro("read_%s" % type, "ptr", "((%s_t)%s(*((%s_t *)(ptr)))" % (type, swap, utype)) + writer.macro("read_%s" % type, "ptr", "((%s_t)%s(*((%s_t *)(ptr))))" % (type, swap, utype)) writer.macro("write_%s" % type, "ptr, val", "*(%s_t *)(ptr) = %s((%s_t)val)" % (utype, swap, utype)) writer.writeln("#else") for size in [8, 16, 32, 64]: From 543faaf796c07e7948830b42def13b3854e958a8 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Tue, 24 Apr 2012 08:34:30 +0300 Subject: [PATCH 109/147] video streaming: add support for frames of different sizes rhbz #813826, #815426 Add SPICE_MSG_DISPLAY_STREAM_DATA_SIZED, for stream_data message that also contains the size and destination box of the data. The server can send such messages only to clients with SPICE_DISPLAY_CAP_SIZED_STREAM. --- spice.proto | 19 ++++++++++++++++--- spice1.proto | 8 ++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/spice.proto b/spice.proto index 513fe87..71be9ac 100644 --- a/spice.proto +++ b/spice.proto @@ -591,6 +591,11 @@ struct String { } u @anon; }; +struct StreamDataHeader { + uint32 id; + uint32 multi_media_time; +}; + channel DisplayChannel : BaseChannel { server: message { @@ -637,10 +642,9 @@ channel DisplayChannel : BaseChannel { } stream_create = 122; message { - uint32 id; - uint32 multi_media_time; + StreamDataHeader base; uint32 data_size; - uint8 data[data_size] @end @nomarshal; + uint8 data[data_size] @end @nomarshal; } stream_data; message { @@ -785,6 +789,15 @@ channel DisplayChannel : BaseChannel { uint32 surface_id; } @ctype(SpiceMsgSurfaceDestroy) surface_destroy; + message { + StreamDataHeader base; + uint32 width; + uint32 height; + Rect dest; + uint32 data_size; + uint8 data[data_size] @end @nomarshal; + } stream_data_sized; + client: message { uint8 pixmap_cache_id; diff --git a/spice1.proto b/spice1.proto index fa2524b..2ed1058 100644 --- a/spice1.proto +++ b/spice1.proto @@ -533,6 +533,11 @@ struct String { } u @anon; }; +struct StreamDataHeader { + uint32 id; + uint32 multi_media_time; +}; + channel DisplayChannel : BaseChannel { server: message { @@ -580,8 +585,7 @@ channel DisplayChannel : BaseChannel { } stream_create = 122; message { - uint32 id; - uint32 multi_media_time; + StreamDataHeader base; uint32 data_size; uint32 pad_size @zero; uint8 data[data_size] @end @nomarshal; From 341f33cbdb77ee8fd804dca0f66b8ddb3bc3602c Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Thu, 14 Jun 2012 11:43:20 +0300 Subject: [PATCH 110/147] python_modules/ptypes.py/EnumBaseType.c_enumname: add missing return to fix broken enums generation Fixes the resulting enums.h from the invocation of: ./spice_codegen.py --generate-enums spice.proto spice-protocol/spice/enums.h Right now any enum will contain None as the enum members, with this fix it will contain the real enum members, i.e. SPICE_FOO. --- python_modules/ptypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 132ba99..d9fbfe2 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -234,7 +234,7 @@ class EnumBaseType(Type): return codegen.prefix_camel(self.name) def c_enumname(self, value): - self.c_enumname_by_name(self.names[value]) + return self.c_enumname_by_name(self.names[value]) def c_enumname_by_name(self, name): if self.has_attr("prefix"): From 8738ce1c5eaf4a1c01a21bcf7b90b7f57ffd5c73 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Thu, 14 Jun 2012 12:40:47 +0300 Subject: [PATCH 111/147] smartcard: build fixes for spice server Define different enums that have a SPICE_ prefix to not conflict with same value enums from libcacard/vsccard_common.h, and continue to use the same SPICE_MSG_SMARTCARD_DATA and SPICE_MSGC_SMARTCARD_DATA enum that is used by the server and clients (spice-gtk, spicec) alike. --- spice.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spice.proto b/spice.proto index 71be9ac..a86bfa7 100644 --- a/spice.proto +++ b/spice.proto @@ -1139,7 +1139,7 @@ enum32 vsc_message_type { APDU, Flush, FlushComplete -} @prefix(VSC_); +}; struct VscMessageHeader { vsc_message_type type; @@ -1170,7 +1170,7 @@ channel SmartcardChannel : BaseChannel { uint32 reader_id; uint32 length; uint8 data[] @end; - } @ctype(SpiceMsgSmartcard) msg = 101; + } @ctype(SpiceMsgSmartcard) data = 101; client: message { @@ -1184,7 +1184,7 @@ channel SmartcardChannel : BaseChannel { case Error: VscMessageError error; } u @anon; - } @ctype(SpiceMsgcSmartcard) msg = 101; + } @ctype(SpiceMsgcSmartcard) data = 101; message { vsc_message_type type; From e85c454a1e659467ff8b95bb602df2dff899461c Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Thu, 7 Jun 2012 15:04:08 +0300 Subject: [PATCH 112/147] support multiple monitors in single display channel See spice-protocol commit for details: da908f89b581fd4725da997fdaea209f8e6548f6 support multiple monitors on a single display channel --- spice.proto | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spice.proto b/spice.proto index a86bfa7..1fdead9 100644 --- a/spice.proto +++ b/spice.proto @@ -596,6 +596,16 @@ struct StreamDataHeader { uint32 multi_media_time; }; +struct Head { + uint32 id; + uint32 surface_id; + uint32 width; + uint32 height; + uint32 x; + uint32 y; + uint32 flags; +}; + channel DisplayChannel : BaseChannel { server: message { @@ -798,6 +808,12 @@ channel DisplayChannel : BaseChannel { uint8 data[data_size] @end @nomarshal; } stream_data_sized; + message { + uint16 count; + uint16 max_allowed; + Head heads[count] @end; + } monitors_config; + client: message { uint8 pixmap_cache_id; From b3eba427bfc9a1c8c23b5ebc859263695435d2a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Sandmann=20Pedersen?= Date: Fri, 18 May 2012 17:01:29 -0400 Subject: [PATCH 113/147] Add support for A8 images to the LZ routines This format is needed to add Render support to the X driver, so we need the ability to compress and decompress it. --- spice.proto | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/spice.proto b/spice.proto index 1fdead9..29d6a8b 100644 --- a/spice.proto +++ b/spice.proto @@ -26,6 +26,15 @@ struct Rect { int32 right; }; +struct Transform { + uint32 t00; + uint32 t01; + uint32 t02; + uint32 t10; + uint32 t11; + uint32 t12; +}; + enum32 link_err { OK, ERROR, @@ -52,6 +61,28 @@ flags32 migrate_flags { NEED_DATA_TRANSFER } @prefix(SPICE_MIGRATE_); +flags32 composite_flags { + OP0, OP1, OP2, OP3, OP4, OP5, OP6, OP7, + SRC_FILTER0, SRC_FILTER1, SRC_FILTER2, + MASK_FILTER0, MASK_FITLER1, MASK_FILTER2, + + SRC_REPEAT0, SRC_REPEAT1, + MASK_REPEAT0, MASK_REPEAT1, + COMPONENT_ALPHA, + + HAS_MASK, + HAS_SRC_TRANSFORM, + HAS_MASK_TRANSFORM, + + /* These are used to override the formats given in the images. For + * example, if the mask image has format a8r8g8b8, but MASK_OPAQUE + * is set, the image should be treated as if it were x8r8g8b8 + */ + SOURCE_OPAQUE, + MASK_OPAQUE, + DEST_OPAQUE, +} @prefix(SPICE_COMPOSITE_); + enum32 notify_severity { INFO, WARN, @@ -321,7 +352,8 @@ enum8 bitmap_fmt { 16BIT, /* 0555 mode */ 24BIT /* 3 byte, brg */, 32BIT /* 4 byte, xrgb in little endian format */, - RGBA /* 4 byte, argb in little endian format */ + RGBA /* 4 byte, argb in little endian format */, + 8BIT_A /* 1 byte, alpha */ }; flags8 bitmap_flags { @@ -814,6 +846,28 @@ channel DisplayChannel : BaseChannel { Head heads[count] @end; } monitors_config; + message { + DisplayBase base; + struct Composite { + composite_flags flags; + Image *src_bitmap; + switch (flags) { + case HAS_MASK: + Image *mask_bitmap; + } a @anon; + switch (flags) { + case HAS_SRC_TRANSFORM: + Transform src_transform; + } b @anon; + switch (flags) { + case HAS_MASK_TRANSFORM: + Transform mask_transform; + } c @anon; + Point16 src_origin; + Point16 mask_origin; + } data; + } draw_composite; + client: message { uint8 pixmap_cache_id; From 6838a4fed07220425fdbcde1b5a80f87c5093fb7 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Wed, 8 Aug 2012 11:06:40 +0300 Subject: [PATCH 114/147] add SPICE_MSG_MAIN_AGENT_CONNECTED_TOKENS The msg is used for setting the number of allocated client tokens when we notify the client that the agent is attached. --- spice.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spice.proto b/spice.proto index 29d6a8b..271e35d 100644 --- a/spice.proto +++ b/spice.proto @@ -263,6 +263,10 @@ channel MainChannel : BaseChannel { uint8 uuid[16]; } uuid; + message { + uint32 num_tokens; + } agent_connected_tokens; + client: message { uint64 cache_size; From 76712d6afa0afd6be6310f3fd35140dec088c0f1 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Mon, 2 Jul 2012 13:23:32 +0300 Subject: [PATCH 115/147] support seamless migration see spice-protocol for more details commit 3838ad140a046c4ddf42fef58c9727ecfdc09f9f --- spice.proto | 24 +++++++++++++++++++++--- spice1.proto | 10 +++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/spice.proto b/spice.proto index 271e35d..2ef4dd6 100644 --- a/spice.proto +++ b/spice.proto @@ -193,15 +193,19 @@ struct ChannelId { uint8 id; }; -channel MainChannel : BaseChannel { - server: - message { +struct DstInfo { uint16 port; uint16 sport; uint32 host_size; uint8 *host_data[host_size] @zero_terminated @marshall @nonnull; uint32 cert_subject_size; uint8 *cert_subject_data[cert_subject_size] @zero_terminated @marshall; +} @ctype(SpiceMigrationDstInfo); + +channel MainChannel : BaseChannel { + server: + message { + DstInfo dst_info; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; @@ -267,6 +271,14 @@ channel MainChannel : BaseChannel { uint32 num_tokens; } agent_connected_tokens; + message { + DstInfo dst_info; + uint32 src_mig_version; + } migrate_begin_seamless; + + Empty migrate_dst_seamless_ack; + Empty migrate_dst_seamless_nack; + client: message { uint64 cache_size; @@ -293,6 +305,12 @@ channel MainChannel : BaseChannel { } @ctype(SpiceMsgcMainAgentTokens) agent_token; Empty migrate_end; + + message { + uint32 src_version; + } migrate_dst_do_seamless; + + Empty migrate_connected_seamless; }; enum8 clip_type { diff --git a/spice1.proto b/spice1.proto index 2ed1058..2d22cdf 100644 --- a/spice1.proto +++ b/spice1.proto @@ -161,9 +161,7 @@ struct ChannelId { uint8 id; }; -channel MainChannel : BaseChannel { - server: - message { +struct DstInfo { uint16 port; uint16 sport; uint32 host_offset @zero; @@ -173,6 +171,12 @@ channel MainChannel : BaseChannel { uint32 pub_key_size @minor(1); uint8 host_data[host_size] @as_ptr @zero_terminated; uint8 pub_key_data[pub_key_size] @minor(1) @as_ptr @zero_terminated; +} @ctype(SpiceMigrationDstInfo); + +channel MainChannel : BaseChannel { + server: + message { + DstInfo dst_info; } @ctype(SpiceMsgMainMigrationBegin) migrate_begin = 101; Empty migrate_cancel; From 911dac9d60c2f1c332ec1ab877417b74de2c60e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 15 Aug 2012 12:52:43 +0300 Subject: [PATCH 116/147] inputs: add a INPUTS_KEY_SCANCODE message Add a new arbitrary keyboard scancodes message. For now, it will be used to avoid unwanted key repeatition when there is jitter in the network and too much time between DOWN and UP messages, instead the client will send the press & release scancode in a sequence. See also: https://bugzilla.redhat.com/show_bug.cgi?id=812347 --- spice.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spice.proto b/spice.proto index 2ef4dd6..315ab91 100644 --- a/spice.proto +++ b/spice.proto @@ -934,6 +934,8 @@ channel InputsChannel : BaseChannel { keyboard_modifier_flags modifiers; } @ctype(SpiceMsgcKeyModifiers) key_modifiers; + Data key_scancode; + message { int32 dx; int32 dy; From f188fb7a890867ebe5eace38021b1f3e1daa230d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Fri, 30 Nov 2012 01:25:44 +0100 Subject: [PATCH 117/147] Add a "port" channel A Spice port channel carry arbitrary data between the Spice client and the Spice server. It may be used to provide additional services on top of a Spice connection. For example, a channel can be associated with the qemu monitor for the client to interact with it, just like any qemu chardev. Or it may be used with various protocols, such as the Spice Controller. A port kind is identified simply by its fqdn, such as org.qemu.monitor, org.spice.spicy.test or org.ovirt.controller... The channel is based on Spicevmc which simply tunnels data between client and server. A few messages have been added: SPICE_MSG_PORT_INIT: Describes the port state and fqdn name, should be sent only once when the client connects. SPICE_MSG_PORT_EVENT: Server port event. SPICE_PORT_EVENT_OPENED and SPICE_PORT_EVENT_CLOSED are typical values when the chardev is opened or closed. SPICE_MSGC_PORT_EVENT: Client port event. --- spice.proto | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/spice.proto b/spice.proto index 315ab91..655352a 100644 --- a/spice.proto +++ b/spice.proto @@ -1309,6 +1309,22 @@ client: channel UsbredirChannel : SpicevmcChannel { }; +channel PortChannel : SpicevmcChannel { + client: + message { + uint8 event; + } event = 201; + server: + message { + uint32 name_size; + uint8 *name[name_size] @zero_terminated @marshall @nonnull; + uint8 opened; + } init = 201; + message { + uint8 event; + } event; +}; + protocol Spice { MainChannel main = 1; DisplayChannel display; @@ -1319,4 +1335,5 @@ protocol Spice { TunnelChannel tunnel; SmartcardChannel smartcard; UsbredirChannel usbredir; + PortChannel port; }; From 1f603c09b8143ca2bed4ccd23b6f0f3bc7e330f9 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Mon, 18 Feb 2013 14:26:25 -0500 Subject: [PATCH 118/147] add stream report messages If the server & client support SPICE_DISPLAY_CAP_STREAM_REPORT, the server first sends SPICE_MSG_DISPLAY_STREAM_ACTIVATE_REPORT. Then, the client periodically sends SPICE_MSGC_DISPLAY_STREAM_REPORT messages that supply the server details about the current quality of the video streaming on the client side. The server analyses the report and adjust the stream parameters accordingly. --- spice.proto | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/spice.proto b/spice.proto index 655352a..97d1e1b 100644 --- a/spice.proto +++ b/spice.proto @@ -890,6 +890,13 @@ channel DisplayChannel : BaseChannel { } data; } draw_composite; + message { + uint32 stream_id; + uint32 unique_id; + uint32 max_window_size; + uint32 timeout_ms; + } stream_activate_report; + client: message { uint8 pixmap_cache_id; @@ -897,6 +904,17 @@ channel DisplayChannel : BaseChannel { uint8 glz_dictionary_id; int32 glz_dictionary_window_size; // in pixels } init = 101; + + message { + uint32 stream_id; + uint32 unique_id; + uint32 start_frame_mm_time; + uint32 end_frame_mm_time; + uint32 num_frames; + uint32 num_drops; + int32 last_frame_delay; + uint32 audio_delay; + } stream_report; }; flags16 keyboard_modifier_flags { From bd3f609b34250135077f7c2348e3a460e4ac2597 Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Wed, 9 Jan 2013 16:11:29 -0500 Subject: [PATCH 119/147] add SPICE_MSG_PLAYBACK_LATENCY SPICE_MSG_PLAYBACK_LATENCY is intended for adjusting the latency of the audio playback. It is used for synchronizing the audio and video playback. The corresponding capability is SPICE_PLAYBACK_CAP_LATENCY. --- spice.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spice.proto b/spice.proto index 97d1e1b..5eede6b 100644 --- a/spice.proto +++ b/spice.proto @@ -1102,6 +1102,10 @@ channel PlaybackChannel : BaseChannel { Empty stop; AudioVolume volume; AudioMute mute; + + message { + uint32 latency_ms; + } latency; }; channel RecordChannel : BaseChannel { From 5dad55f0081688067083f95bd45f4512e602cd93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 10 Sep 2013 12:30:57 +0200 Subject: [PATCH 120/147] proto: comment future surface flags usage --- spice.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spice.proto b/spice.proto index 5eede6b..728178b 100644 --- a/spice.proto +++ b/spice.proto @@ -425,6 +425,8 @@ flags8 string_flags { }; flags32 surface_flags { + /* Adding flags requires some caps check, since old clients only + treat the value as an enum and not as a flag (flag == PRIMARY) */ PRIMARY }; From 23fe54f11274e4117f4bffd033a7a045cb91de47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 10 Sep 2013 23:10:29 +0200 Subject: [PATCH 121/147] proto: add fake last message in base channel Make it explicit that 100 is the last value of the base channel messages. This allows clients to use the generated enum value too. --- spice.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spice.proto b/spice.proto index 728178b..04e7ea4 100644 --- a/spice.proto +++ b/spice.proto @@ -166,6 +166,8 @@ channel BaseChannel { Data list; /* the msg body is SpiceSubMessageList */ + Empty base_last = 100; + client: message { uint32 generation; From f3a47cc903a4a0137264431d0ebb4c01f2959969 Mon Sep 17 00:00:00 2001 From: Jonathon Jongsma Date: Thu, 12 Sep 2013 12:11:02 -0500 Subject: [PATCH 122/147] codegen: Add a --generate-wireshark-dissector option The wireshark protocol dissector is a bit out-of-date. Several new channel types and enums have been added. It would be nice if these values (and the translation between the value and the name) could be automatically generated so that updating the dissector was a slightly less manual process. This patch adds a commandline switch which generates both the enums and value-name lists in the format that wireshark expects. --- python_modules/ptypes.py | 17 +++++++ spice_codegen.py | 97 +++++++++++++++++++++++++++------------- 2 files changed, 82 insertions(+), 32 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index d9fbfe2..2bfbf0d 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -247,6 +247,23 @@ class EnumBaseType(Type): def get_fixed_nw_size(self): return self.bits / 8 + # generates a value-name table suitable for use with the wireshark protocol + # dissector + def c_describe(self, writer): + writer.write("static const value_string %s_vs[] = " % codegen.prefix_underscore_lower(self.name)) + writer.begin_block() + values = self.names.keys() + values.sort() + for i in values: + writer.write("{ ") + writer.write(self.c_enumname(i)) + writer.write(", \"%s\" }," % self.names[i]) + writer.newline() + writer.write("{ 0, NULL }") + writer.end_block(semicolon=True) + writer.newline() + + class EnumType(EnumBaseType): def __init__(self, bits, name, enums, attribute_list): Type.__init__(self) diff --git a/spice_codegen.py b/spice_codegen.py index e9e64c0..1cf1697 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -10,38 +10,73 @@ from python_modules import codegen from python_modules import demarshal from python_modules import marshal -def write_channel_enums(writer, channel, client): +def write_channel_enums(writer, channel, client, describe): messages = filter(lambda m : m.channel == channel, \ channel.client_messages if client else channel.server_messages) if len(messages) == 0: return - writer.begin_block("enum") - i = 0 if client: prefix = [ "MSGC" ] else: prefix = [ "MSG" ] if channel.member_name: prefix.append(channel.member_name.upper()) + if not describe: + writer.begin_block("enum") + else: + writer.begin_block("static const value_string %s_vs[] = " % (codegen.prefix_underscore_lower(*[x.lower() for x in prefix]))) + i = 0 prefix.append(None) # To be replaced with name for m in messages: prefix[-1] = m.name.upper() enum = codegen.prefix_underscore_upper(*prefix) - if m.value == i: - writer.writeln("%s," % enum) - i = i + 1 + if describe: + writer.writeln("{ %s, \"%s %s\" }," % (enum, "Client" if client else "Server", m.name.upper())) else: - writer.writeln("%s = %s," % (enum, m.value)) - i = m.value + 1 - if channel.member_name: - prefix[-1] = prefix[-2] - prefix[-2] = "END" - writer.newline() - writer.writeln("%s" % (codegen.prefix_underscore_upper(*prefix))) + if m.value == i: + writer.writeln("%s," % enum) + i = i + 1 + else: + writer.writeln("%s = %s," % (enum, m.value)) + i = m.value + 1 + if describe: + writer.writeln("{ 0, NULL }"); + else: + if channel.member_name: + prefix[-1] = prefix[-2] + prefix[-2] = "END" + writer.newline() + writer.writeln("%s" % (codegen.prefix_underscore_upper(*prefix))) writer.end_block(semicolon=True) writer.newline() -def write_enums(writer): +def write_channel_type_enum(writer, describe=False): + i = 0 + if describe: + writer.begin_block("static const value_string channel_types_vs[] =") + else: + writer.begin_block("enum") + for c in proto.channels: + enum = codegen.prefix_underscore_upper("CHANNEL", c.name.upper()) + if describe: + writer.writeln("{ %s, \"%s\" }," % (enum, c.name.upper())) + else: + if c.value == i: + writer.writeln("%s," % enum) + i = i + 1 + else: + writer.writeln("%s = %s," % (enum, c.value)) + i = c.value + 1 + writer.newline() + if describe: + writer.writeln("{ 0, NULL }") + else: + writer.writeln("SPICE_END_CHANNEL") + writer.end_block(semicolon=True) + writer.newline() + + +def write_enums(writer, describe=False): writer.writeln("#ifndef _H_SPICE_ENUMS") writer.writeln("#define _H_SPICE_ENUMS") writer.newline() @@ -52,27 +87,22 @@ def write_enums(writer): for t in ptypes.get_named_types(): if isinstance(t, ptypes.EnumBaseType): t.c_define(writer) + if describe: + t.c_describe(writer) - i = 0 - writer.begin_block("enum") - for c in proto.channels: - enum = codegen.prefix_underscore_upper("CHANNEL", c.name.upper()) - if c.value == i: - writer.writeln("%s," % enum) - i = i + 1 - else: - writer.writeln("%s = %s," % (enum, c.value)) - i = c.value + 1 - writer.newline() - writer.writeln("SPICE_END_CHANNEL") - writer.end_block(semicolon=True) - writer.newline() + write_channel_type_enum(writer) + if (describe): + write_channel_type_enum(writer, True) for c in ptypes.get_named_types(): if not isinstance(c, ptypes.ChannelType): continue - write_channel_enums(writer, c, False) - write_channel_enums(writer, c, True) + write_channel_enums(writer, c, False, False) + if describe: + write_channel_enums(writer, c, False, describe) + write_channel_enums(writer, c, True, False) + if describe: + write_channel_enums(writer, c, True, describe) writer.writeln("#endif /* _H_SPICE_ENUMS */") @@ -80,6 +110,9 @@ parser = OptionParser(usage="usage: %prog [options] Date: Tue, 22 Oct 2013 10:38:48 -0500 Subject: [PATCH 123/147] Be explicit about spice-common license Use same license as spice-gtk and spice modules (LGPL 2.1) since those licenses applied to the spice-common submodule in the past. This makes it more clear that if you use spice-common separately, the license is still LGPL. Also mention license and copyright in generated files. --- spice_codegen.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/spice_codegen.py b/spice_codegen.py index 1cf1697..d35d949 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -80,8 +80,6 @@ def write_enums(writer, describe=False): writer.writeln("#ifndef _H_SPICE_ENUMS") writer.writeln("#define _H_SPICE_ENUMS") writer.newline() - writer.comment("Generated from %s, don't edit" % writer.options["source"]).newline() - writer.newline() # Define enums for t in ptypes.get_named_types(): @@ -173,10 +171,31 @@ writer = codegen.CodeWriter() writer.header = codegen.CodeWriter() writer.set_option("source", os.path.basename(proto_file)) +license = """/* + Copyright (C) 2013 Red Hat, Inc. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, see . +*/ + +""" + writer.public_prefix = options.prefix writer.writeln("/* this is a file autogenerated by spice_codegen.py */") +writer.write(license) writer.header.writeln("/* this is a file autogenerated by spice_codegen.py */") +writer.header.write(license) if not options.header and not options.generate_enums: writer.writeln("#ifdef HAVE_CONFIG_H") writer.writeln("#include ") From ee94b6b5f2775e6522c800711d6c432a7cb6015c Mon Sep 17 00:00:00 2001 From: Jeremy White Date: Sat, 30 Nov 2013 09:18:48 -0600 Subject: [PATCH 124/147] Add support for the Opus codec. Signed-off-by: Jeremy White --- spice.proto | 1 + spice1.proto | 1 + 2 files changed, 2 insertions(+) diff --git a/spice.proto b/spice.proto index 04e7ea4..67b3803 100644 --- a/spice.proto +++ b/spice.proto @@ -1067,6 +1067,7 @@ enum16 audio_data_mode { INVALID, RAW, CELT_0_5_1, + OPUS, }; enum16 audio_fmt { diff --git a/spice1.proto b/spice1.proto index 2d22cdf..67eb0e6 100644 --- a/spice1.proto +++ b/spice1.proto @@ -876,6 +876,7 @@ enum32 audio_data_mode { INVALID, RAW, CELT_0_5_1, + OPUS, }; enum32 audio_fmt { From 6e5ea8d802ac8b2b92cb6f483a308ee196efe29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Fri, 10 Jan 2014 17:01:37 +0100 Subject: [PATCH 125/147] spice.proto: add webdav channel This channel provides a webdav server (rfc4918). This allows various guest or remote system that support webdav to access a folder shared by the client (some agent can be used to proxy the requests on a local port for example). The webdav server may also be accessed by an hypervisor as a remote filesystem interface, which can then be accessed by the guest via other means (fs/fat emulation, mtp device, etc) Due to the usage of a single channel stream and the need for concurrent requests, webdav clients streams are multiplexed. Each client stream is framed within 64k max messages (in little-endian) int64 client_id uint16 size char data[size] A new client_id indicates a new connection. A new communication stream with the webdav server should be started. A client stream message of size 0 indicates a disconnection of client_id. This multiplexed communication happens over the channel "data" message. Only when the port is opened may the communication be started. A closed port event should close all currently known multiplexed connections. Why WebDAV? webdav is supported natively by various OS for a long time (circa Windows XP). It has several open-source implementations and a variety of tools exist. A webdav implementation can be tested and used without a Spice server or any virtualization (this also permit sharing the implementation with other projects in the future, such as GNOME). It is an IETF open standard and thus thoroughly specified. The basic requirements for an efficient remote filesystem are provided by the standard (pipelining, concurrency, caching, copy/move, partial io, compression, locking ...) While other features are easily possible via extensions to the protocol (common ones are executable attributes, or searching for example). Given the requirements, and the popularity of http/webdav, I believe it is the best candidate for Spice remote filesystem support. Other alternatives (adhoc, p9, smb2, sftp) have been studied and discarded so far since they do not match in term of features or requirements. --- spice.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spice.proto b/spice.proto index 67b3803..316d287 100644 --- a/spice.proto +++ b/spice.proto @@ -1352,6 +1352,9 @@ channel PortChannel : SpicevmcChannel { } event; }; +channel WebDAVChannel : PortChannel { +}; + protocol Spice { MainChannel main = 1; DisplayChannel display; @@ -1363,4 +1366,5 @@ protocol Spice { SmartcardChannel smartcard; UsbredirChannel usbredir; PortChannel port; + WebDAVChannel webdav; }; From ef14521ac5218ca876e29173470fdb41a0cc821c Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 19 Mar 2014 15:41:07 +0100 Subject: [PATCH 126/147] Use #include "common/..." in (de)marshallers Now that they are created in $builddir, their includes will need to refer to files in $srcdir, which can be different. It's cleaner to add -I $(top_srcdir)/spice-common/ to modules using spice-common rather than having -I $(top_srcdir)/spice-common/common which would could create header collisions. --- python_modules/demarshal.py | 2 +- python_modules/marshal.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index cf48d74..794cd25 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1250,7 +1250,7 @@ def write_includes(writer): writer.writeln("#include ") writer.writeln("#include ") writer.writeln("#include ") - writer.writeln('#include "mem.h"') + writer.writeln('#include "common/mem.h"') writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 4020799..ae5fe31 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -4,7 +4,7 @@ import codegen def write_includes(writer): writer.header.writeln("#include ") - writer.header.writeln('#include "marshaller.h"') + writer.header.writeln('#include "common/marshaller.h"') writer.header.newline() writer.header.writeln("#ifndef _GENERATED_HEADERS_H") writer.header.writeln("#define _GENERATED_HEADERS_H") @@ -15,7 +15,7 @@ def write_includes(writer): writer.writeln("#include ") writer.writeln("#include ") writer.writeln("#include ") - writer.writeln('#include "marshaller.h"') + writer.writeln('#include "common/marshaller.h"') writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") From 06a99ce30b113566b40453e42752d45582b3a9f7 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Tue, 25 Mar 2014 16:32:16 +0100 Subject: [PATCH 127/147] marshaller: Use #include <> for headers in $srcdir/common Since the (de)marshallers are now generated in $builddir and not in $srcdir, when these generated files include a file located in $srcdir/common, the compiler will find them thanks to a -I directive, so it makes more sense to use <> rather than "" when including them. --- python_modules/demarshal.py | 2 +- spice_codegen.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 794cd25..b7e51dc 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1250,7 +1250,7 @@ def write_includes(writer): writer.writeln("#include ") writer.writeln("#include ") writer.writeln("#include ") - writer.writeln('#include "common/mem.h"') + writer.writeln('#include ') writer.newline() writer.writeln("#ifdef _MSC_VER") writer.writeln("#pragma warning(disable:4101)") diff --git a/spice_codegen.py b/spice_codegen.py index d35d949..c8376cc 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -209,8 +209,8 @@ if options.print_error: if options.includes: for i in options.includes: - writer.header.writeln('#include "%s"' % i) - writer.writeln('#include "%s"' % i) + writer.header.writeln('#include <%s>' % i) + writer.writeln('#include <%s>' % i) if options.generate_enums or options.generate_dissector: write_enums(writer, options.generate_dissector) From 3916b9d813e716d0c8c03a2be82dd169e4461e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 16 Apr 2014 13:23:12 +0200 Subject: [PATCH 128/147] demarshal: prefix variable name with parent names Avoid naming clash if parent structures have the same name https://bugzilla.redhat.com/show_bug.cgi?id=1021995 --- python_modules/demarshal.py | 47 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index b7e51dc..ead776d 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -138,22 +138,25 @@ class ItemInfo: return self.position class MemberItemInfo(ItemInfo): - def __init__(self, member, container, start): + def __init__(self, member, mprefix, container, start): if not member.is_switch(): self.type = member.member_type self.prefix = member.name + if mprefix: + mprefix = mprefix + "_" + self.prefix = mprefix + self.prefix self.subprefix = member.name - self.position = "(%s + %s)" % (start, container.get_nw_offset(member, "", "__nw_size")) + self.position = "(%s + %s)" % (start, container.get_nw_offset(member, mprefix or "", "__nw_size")) self.member = member -def write_validate_switch_member(writer, container, switch_member, scope, parent_scope, start, +def write_validate_switch_member(writer, mprefix, container, switch_member, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): var = container.lookup_member(switch_member.variable) var_type = var.member_type v = write_read_primitive(writer, start, container, switch_member.variable, parent_scope) - item = MemberItemInfo(switch_member, container, start) + item = MemberItemInfo(switch_member, mprefix, container, start) first = True for c in switch_member.cases: @@ -487,7 +490,7 @@ def write_validate_item(writer, container, item, scope, parent_scope, start, writer.add_function_variable("uint32_t", saved_size + " = 0") writer.assign(saved_size, item.nw_size()) -def write_validate_member(writer, container, member, parent_scope, start, +def write_validate_member(writer, mprefix, container, member, parent_scope, start, want_nw_size, want_mem_size, want_extra_size): if member.has_attr("virtual"): return @@ -498,10 +501,10 @@ def write_validate_member(writer, container, member, parent_scope, start, else: prefix = "" newline = True - item = MemberItemInfo(member, container, start) + item = MemberItemInfo(member, mprefix, container, start) with writer.block(prefix, newline=newline, comment=member.name) as scope: if member.is_switch(): - write_validate_switch_member(writer, container, member, scope, parent_scope, start, + write_validate_switch_member(writer, mprefix, container, member, scope, parent_scope, start, want_nw_size, want_mem_size, want_extra_size) else: write_validate_item(writer, container, item, scope, parent_scope, start, @@ -526,22 +529,29 @@ def write_validate_member(writer, container, member, parent_scope, start, assert not want_extra_size def write_validate_container(writer, prefix, container, start, parent_scope, want_nw_size, want_mem_size, want_extra_size): + def prefix_m(prefix, m): + name = m.name + if prefix: + name = prefix + "_" + name + return name + for m in container.members: sub_want_nw_size = want_nw_size and not m.is_fixed_nw_size() sub_want_mem_size = m.is_extra_size() and want_mem_size sub_want_extra_size = not m.is_extra_size() and m.contains_extra_size() - defs = ["size_t"] + name = prefix_m(prefix, m) if sub_want_nw_size: - defs.append (m.name + "__nw_size") + + defs.append (name + "__nw_size") if sub_want_mem_size: - defs.append (m.name + "__mem_size") + defs.append (name + "__mem_size") if sub_want_extra_size: - defs.append (m.name + "__extra_size") + defs.append (name + "__extra_size") if sub_want_nw_size or sub_want_mem_size or sub_want_extra_size: parent_scope.variable_def(*defs) - write_validate_member(writer, container, m, parent_scope, start, + write_validate_member(writer, prefix, container, m, parent_scope, start, sub_want_nw_size, sub_want_mem_size, sub_want_extra_size) writer.newline() @@ -558,8 +568,9 @@ def write_validate_container(writer, prefix, container, start, parent_scope, wan nm_sum = str(size) for m in container.members: + name = prefix_m(prefix, m) if not m.is_fixed_nw_size(): - nm_sum = nm_sum + " + " + m.name + "__nw_size" + nm_sum = nm_sum + " + " + name + "__nw_size" writer.assign(nw_size, nm_sum) @@ -571,10 +582,11 @@ def write_validate_container(writer, prefix, container, start, parent_scope, wan mem_sum = container.sizeof() for m in container.members: + name = prefix_m(prefix, m) if m.is_extra_size(): - mem_sum = mem_sum + " + " + m.name + "__mem_size" + mem_sum = mem_sum + " + " + name + "__mem_size" elif m.contains_extra_size(): - mem_sum = mem_sum + " + " + m.name + "__extra_size" + mem_sum = mem_sum + " + " + name + "__extra_size" writer.assign(mem_size, mem_sum) @@ -586,10 +598,11 @@ def write_validate_container(writer, prefix, container, start, parent_scope, wan extra_sum = [] for m in container.members: + name = prefix_m(prefix, m) if m.is_extra_size(): - extra_sum.append(m.name + "__mem_size") + extra_sum.append(name + "__mem_size") elif m.contains_extra_size(): - extra_sum.append(m.name + "__extra_size") + extra_sum.append(name + "__extra_size") writer.assign(extra_size, codegen.sum_array(extra_sum)) class DemarshallingDestination: From 744675b424a9db5e6fc26440dcfab961fd1d284e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Sep 2014 11:51:45 +0200 Subject: [PATCH 129/147] python: Fix -Wunused-parameter Although the most part of the parameters marked as unused are actually being used for a few functions, a bunch of warnings can be seen when the code is compiled with "-Wall -Wextra". As adding the unused attribute means that the variable/parameter is meant to be *possibly* unused, we're safe adding it in the generated code, even for used variables/parameters. --- python_modules/demarshal.py | 12 ++++++------ python_modules/marshal.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index ead776d..109f5e6 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -199,7 +199,7 @@ def write_validate_struct_function(writer, struct): writer.set_is_generated("validator", validate_function) writer = writer.function_helper() - scope = writer.function(validate_function, "static intptr_t", "uint8_t *message_start, uint8_t *message_end, uint64_t offset, int minor") + scope = writer.function(validate_function, "static intptr_t", "uint8_t *message_start, uint8_t *message_end, uint64_t offset, SPICE_GNUC_UNUSED int minor") scope.variable_def("uint8_t *", "start = message_start + offset") scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos") scope.variable_def("size_t", "mem_size", "nw_size") @@ -773,7 +773,7 @@ def write_parse_ptr_function(writer, target_type): writer.set_is_generated("parser", parse_function) writer = writer.function_helper() - scope = writer.function(parse_function, "static uint8_t *", "uint8_t *message_start, uint8_t *message_end, uint8_t *struct_data, PointerInfo *this_ptr_info, int minor") + scope = writer.function(parse_function, "static uint8_t *", "uint8_t *message_start, SPICE_GNUC_UNUSED uint8_t *message_end, uint8_t *struct_data, PointerInfo *this_ptr_info, SPICE_GNUC_UNUSED int minor") scope.variable_def("uint8_t *", "in = message_start + this_ptr_info->offset") scope.variable_def("uint8_t *", "end") @@ -1011,7 +1011,7 @@ def write_nofree(writer): if writer.is_generated("helper", "nofree"): return writer = writer.function_helper() - scope = writer.function("nofree", "static void", "uint8_t *data") + scope = writer.function("nofree", "static void", "SPICE_GNUC_UNUSED uint8_t *data") writer.end_block() def write_msg_parser(writer, message): @@ -1032,7 +1032,7 @@ def write_msg_parser(writer, message): writer.ifdef(message.attributes["ifdef"][0]) parent_scope = writer.function(function_name, "uint8_t *", - "uint8_t *message_start, uint8_t *message_end, int minor, size_t *size, message_destructor_t *free_message", True) + "uint8_t *message_start, uint8_t *message_end, SPICE_GNUC_UNUSED int minor, size_t *size, message_destructor_t *free_message", True) parent_scope.variable_def("SPICE_GNUC_UNUSED uint8_t *", "pos") parent_scope.variable_def("uint8_t *", "start = message_start") parent_scope.variable_def("uint8_t *", "data = NULL") @@ -1134,7 +1134,7 @@ def write_channel_parser(writer, channel, server): writer.ifdef(channel.attributes["ifdef"][0]) scope = writer.function(function_name, "static uint8_t *", - "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") + "uint8_t *message_start, uint8_t *message_end, uint16_t message_type, SPICE_GNUC_UNUSED int minor, size_t *size_out, message_destructor_t *free_message") helpers = writer.function_helper() @@ -1230,7 +1230,7 @@ def write_full_protocol_parser(writer, is_server): function_name = "spice_parse_reply" scope = writer.function(function_name + writer.public_prefix, "uint8_t *", - "uint8_t *message_start, uint8_t *message_end, uint32_t channel, uint16_t message_type, int minor, size_t *size_out, message_destructor_t *free_message") + "uint8_t *message_start, uint8_t *message_end, uint32_t channel, uint16_t message_type, SPICE_GNUC_UNUSED int minor, size_t *size_out, message_destructor_t *free_message") scope.variable_def("spice_parse_channel_func_t", "func" ) if is_server: diff --git a/python_modules/marshal.py b/python_modules/marshal.py index ae5fe31..97d02e9 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -353,7 +353,7 @@ def write_message_marshaller(writer, message, is_server, private): scope = writer.function(function_name, "static void" if private else "void", - "SpiceMarshaller *m, %s *msg" % message.c_type() + names_args) + "SPICE_GNUC_UNUSED SpiceMarshaller *m, SPICE_GNUC_UNUSED %s *msg" % message.c_type() + names_args) scope.variable_def("SPICE_GNUC_UNUSED SpiceMarshaller *", "m2") for n in names: From cb37de352012da27268882eb24784ba39e834712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Sep 2014 15:14:54 +0200 Subject: [PATCH 130/147] python: Fix -Wsign-compare The return of the get_array_size() is used as a limit in a loop, being compared with an unsigned index (indexes are always unsigned). To avoid the -Wsign-compare warning, let's cast the return of the get_array_size() to unsigned and make GCC happier. --- python_modules/marshal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 97d02e9..1eda1ba 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -162,11 +162,11 @@ def get_array_size(array, container_src): rows_v = container_src.get_ref(rows) # TODO: Handle multiplication overflow if bpp == 8: - return "(%s * %s)" % (width_v, rows_v) + return "(unsigned) (%s * %s)" % (width_v, rows_v) elif bpp == 1: - return "(((%s + 7) / 8 ) * %s)" % (width_v, rows_v) + return "(unsigned) (((%s + 7) / 8 ) * %s)" % (width_v, rows_v) else: - return "(((%s * %s + 7) / 8 ) * %s)" % (bpp, width_v, rows_v) + return "(unsigned) (((%s * %s + 7) / 8 ) * %s)" % (bpp, width_v, rows_v) elif array.is_bytes_length(): return container_src.get_ref(array.size[2]) else: From 77ce36426f26cd3dcf8bb06a0511f365c74003dc Mon Sep 17 00:00:00 2001 From: Javier Celaya Date: Wed, 22 Oct 2014 09:58:29 +0200 Subject: [PATCH 131/147] Add LZ4 image compression support. - Add a new LZ4 image type to spice.proto. - Add canvas_get_lz4() to common_canvas_base, to get a pixmap from an lz4 image. - Add an enable-lz4 switch to the configure script, disabled by default. --- spice.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spice.proto b/spice.proto index 316d287..01493c9 100644 --- a/spice.proto +++ b/spice.proto @@ -358,6 +358,7 @@ enum8 image_type { FROM_CACHE_LOSSLESS, ZLIB_GLZ_RGB, JPEG_ALPHA, + LZ4, }; flags8 image_flags { @@ -568,6 +569,8 @@ struct Image { BinaryData lz_rgb; case JPEG: BinaryData jpeg; + case LZ4: + BinaryData lz4; case LZ_PLT: LZPLTData lz_plt; case ZLIB_GLZ_RGB: From e919337980c45fb4bc907cf40cdd106fb8f32d92 Mon Sep 17 00:00:00 2001 From: Alexander Wauck Date: Tue, 31 Mar 2015 12:44:09 -0500 Subject: [PATCH 132/147] Make spice_codegen.py work on both Python 2 and 3 This is a new version of my previous patch that does not include six.py. It's still kind of big, but at least it's all spice-common changes now. There are also a few other fixes that Christophe brought to my attention. Note that six now needs to be installed on the system (python-six on Fedora and Debian, six on PyPI). This *should* be enough to make spice_codegen.py work on both Python 2 and Python 3. The major changes are as follows: * cStringIO.StringIO -> io.StringIO * str vs. unicode updates (io.StringIO doesn't like str) * integer division * foo.has_key(bar) -> bar in foo * import internal_thing -> from . import internal_thing * removed from __future__ import with_statement (might break Python 2.5?) * changed some lambdas to list comprehensions (done by 2to3) * cast some_dict.keys() to list where needed (e.g. for sorting) * use normal type names with isinstance instead of types.WhateverType Signed-off-by: Alexander Wauck --- python_modules/codegen.py | 28 ++++++++------- python_modules/demarshal.py | 12 +++---- python_modules/marshal.py | 18 +++++----- python_modules/ptypes.py | 66 +++++++++++++++++----------------- python_modules/spice_parser.py | 13 +++---- spice_codegen.py | 10 ++++-- 6 files changed, 78 insertions(+), 69 deletions(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 009cf95..55f513b 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -1,5 +1,6 @@ -from __future__ import with_statement -from cStringIO import StringIO + +import six +from io import StringIO def camel_to_underscores(s, upper = False): res = "" @@ -85,10 +86,10 @@ class CodeWriter: self.options[opt] = value def has_option(self, opt): - return self.options.has_key(opt) + return opt in self.options def set_is_generated(self, kind, name): - if not self.generated.has_key(kind): + if kind not in self.generated: v = {} self.generated[kind] = v else: @@ -96,13 +97,13 @@ class CodeWriter: v[name] = 1 def is_generated(self, kind, name): - if not self.generated.has_key(kind): + if kind not in self.generated: return False v = self.generated[kind] - return v.has_key(name) + return name in v def getvalue(self): - strs = map(lambda writer: writer.getvalue(), self.contents) + strs = [writer.getvalue() for writer in self.contents] return "".join(strs) def get_subwriter(self): @@ -119,21 +120,24 @@ class CodeWriter: return writer def write(self, s): - # Ensure its a string - s = str(s) + # Ensure its a unicode string + if six.PY2: + s = unicode(s) + else: + s = str(s) if len(s) == 0: return if self.at_line_start: for i in range(self.indentation): - self.out.write(" ") + self.out.write(u" ") self.at_line_start = False self.out.write(s) return self def newline(self): - self.out.write("\n") + self.out.write(u"\n") self.at_line_start = True return self @@ -341,7 +345,7 @@ class CodeWriter: self.indentation = indentation def add_function_variable(self, ctype, name): - if self.function_variables.has_key(name): + if name in self.function_variables: assert(self.function_variables[name] == ctype) else: self.function_variables[name] = ctype diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py index 109f5e6..209eafc 100644 --- a/python_modules/demarshal.py +++ b/python_modules/demarshal.py @@ -1,6 +1,6 @@ -from __future__ import with_statement -import ptypes -import codegen + +from . import ptypes +from . import codegen # The handling of sizes is somewhat complex, as there are several types of size: # * nw_size @@ -68,7 +68,7 @@ def write_parser_helpers(writer): scope = writer.function("SPICE_GNUC_UNUSED consume_%s" % type, ctype, "uint8_t **ptr", True) scope.variable_def(ctype, "val") writer.assign("val", "read_%s(*ptr)" % type) - writer.increment("*ptr", size / 8) + writer.increment("*ptr", size // 8) writer.statement("return val") writer.end_block() @@ -1119,7 +1119,7 @@ def write_channel_parser(writer, channel, server): ids2 = ids.copy() while len(ids2) > 0: end = start = min(ids2.keys()) - while ids2.has_key(end): + while end in ids2: del ids2[end] end = end + 1 @@ -1181,7 +1181,7 @@ def write_get_channel_parser(writer, channel_parsers, max_channel, is_server): writer.begin_block() channel = None for i in range(0, max_channel + 1): - if channel_parsers.has_key(i): + if i in channel_parsers: channel = channel_parsers[i][0] if channel.has_attr("ifdef"): writer.ifdef(channel.attributes["ifdef"][0]) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index 1eda1ba..b77b910 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -1,6 +1,6 @@ -from __future__ import with_statement -import ptypes -import codegen + +from . import ptypes +from . import codegen def write_includes(writer): writer.header.writeln("#include ") @@ -112,7 +112,7 @@ def write_marshal_ptr_function(writer, target_type, is_helper=True): names = target_type.get_pointer_names(False) names_args = "" if len(names) > 0: - n = map(lambda name: ", SpiceMarshaller **%s_out" % name, names) + n = [", SpiceMarshaller **%s_out" % name for name in names] names_args = "".join(n) header = writer.header @@ -345,7 +345,7 @@ def write_message_marshaller(writer, message, is_server, private): names = message.get_pointer_names(False) names_args = "" if len(names) > 0: - n = map(lambda name: ", SpiceMarshaller **%s_out" % name, names) + n = [", SpiceMarshaller **%s_out" % name for name in names] names_args = "".join(n) if not private: @@ -383,9 +383,9 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): for m in channel.client_messages: message = m.message_type f = write_message_marshaller(writer, message, is_server, private_marshallers) - if channel.has_attr("ifdef") and not functions.has_key(f): + if channel.has_attr("ifdef") and f not in functions: functions[f] = channel.attributes["ifdef"][0] - elif message.has_attr("ifdef") and not functions.has_key(f): + elif message.has_attr("ifdef") and f not in functions: functions[f] = message.attributes["ifdef"][0] else: functions[f] = True @@ -393,9 +393,9 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): for m in channel.server_messages: message = m.message_type f = write_message_marshaller(writer, message, is_server, private_marshallers) - if channel.has_attr("ifdef") and not functions.has_key(f): + if channel.has_attr("ifdef") and f not in functions: functions[f] = channel.attributes["ifdef"][0] - elif message.has_attr("ifdef") and not functions.has_key(f): + elif message.has_attr("ifdef") and f not in functions: functions[f] = message.attributes["ifdef"][0] else: functions[f] = True diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 2bfbf0d..d031d09 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -1,4 +1,4 @@ -import codegen +from . import codegen import types _types_by_name = {} @@ -7,7 +7,7 @@ _types = [] default_pointer_size = 4 def type_exists(name): - return _types_by_name.has_key(name) + return name in _types_by_name def lookup_type(name): return _types_by_name[name] @@ -24,7 +24,7 @@ class FixedSize: self.vals[minor] = val def __add__(self, other): - if isinstance(other, types.IntType): + if isinstance(other, int): other = FixedSize(other) new = FixedSize() @@ -125,13 +125,13 @@ class Type: if self.registred or self.name == None: return self.registred = True - if _types_by_name.has_key(self.name): - raise Exception, "Type %s already defined" % self.name + if self.name in _types_by_name: + raise Exception("Type %s already defined" % self.name) _types.append(self) _types_by_name[self.name] = self def has_attr(self, name): - return self.attributes.has_key(name) + return name in self.attributes class TypeRef(Type): def __init__(self, name): @@ -142,8 +142,8 @@ class TypeRef(Type): return "ref to %s" % (self.name) def resolve(self): - if not _types_by_name.has_key(self.name): - raise Exception, "Unknown type %s" % self.name + if self.name not in _types_by_name: + raise Exception("Unknown type %s" % self.name) return _types_by_name[self.name] def register(self): @@ -168,7 +168,7 @@ class IntegerType(Type): return self.name + "_t" def get_fixed_nw_size(self): - return self.bits / 8 + return self.bits // 8 def is_primitive(self): return True @@ -245,14 +245,14 @@ class EnumBaseType(Type): return True def get_fixed_nw_size(self): - return self.bits / 8 + return self.bits // 8 # generates a value-name table suitable for use with the wireshark protocol # dissector def c_describe(self, writer): writer.write("static const value_string %s_vs[] = " % codegen.prefix_underscore_lower(self.name)) writer.begin_block() - values = self.names.keys() + values = list(self.names.keys()) values.sort() for i in values: writer.write("{ ") @@ -281,7 +281,7 @@ class EnumType(EnumBaseType): value = last + 1 last = value - assert not names.has_key(value) + assert value not in names names[value] = name values[name] = value @@ -298,7 +298,7 @@ class EnumType(EnumBaseType): writer.write("typedef enum ") writer.write(self.c_name()) writer.begin_block() - values = self.names.keys() + values = list(self.names.keys()) values.sort() current_default = 0 for i in values: @@ -335,7 +335,7 @@ class FlagsType(EnumBaseType): value = last + 1 last = value - assert not names.has_key(value) + assert value not in names names[value] = name values[name] = value @@ -352,7 +352,7 @@ class FlagsType(EnumBaseType): writer.write("typedef enum ") writer.write(self.c_name()) writer.begin_block() - values = self.names.keys() + values = list(self.names.keys()) values.sort() mask = 0 for i in values: @@ -392,26 +392,26 @@ class ArrayType(Type): return self def is_constant_length(self): - return isinstance(self.size, types.IntType) + return isinstance(self.size, int) def is_remaining_length(self): - return isinstance(self.size, types.StringType) and len(self.size) == 0 + return isinstance(self.size, str) and len(self.size) == 0 def is_identifier_length(self): - return isinstance(self.size, types.StringType) and len(self.size) > 0 + return isinstance(self.size, str) and len(self.size) > 0 def is_image_size_length(self): - if isinstance(self.size, types.IntType) or isinstance(self.size, types.StringType): + if isinstance(self.size, int) or isinstance(self.size, str): return False return self.size[0] == "image_size" def is_bytes_length(self): - if isinstance(self.size, types.IntType) or isinstance(self.size, types.StringType): + if isinstance(self.size, int) or isinstance(self.size, str): return False return self.size[0] == "bytes" def is_cstring_length(self): - if isinstance(self.size, types.IntType) or isinstance(self.size, types.StringType): + if isinstance(self.size, int) or isinstance(self.size, str): return False return self.size[0] == "cstring" @@ -423,7 +423,7 @@ class ArrayType(Type): def get_fixed_nw_size(self): if not self.is_fixed_nw_size(): - raise Exception, "Not a fixed size type" + raise Exception("Not a fixed size type") return self.element_type.get_fixed_nw_size() * self.size @@ -433,13 +433,13 @@ class ArrayType(Type): return 0 if self.is_constant_length(self): return element_count * self.size - raise Exception, "Pointers in dynamic arrays not supported" + raise Exception("Pointers in dynamic arrays not supported") def get_pointer_names(self, marshalled): element_count = self.element_type.get_num_pointers() if element_count == 0: return [] - raise Exception, "Pointer names in arrays not supported" + raise Exception("Pointer names in arrays not supported") def is_extra_size(self): return self.has_attr("ptr_array") @@ -517,7 +517,7 @@ class Containee: return not self.is_switch() and self.member_type.is_primitive() def has_attr(self, name): - return self.attributes.has_key(name) + return name in self.attributes def has_minor_attr(self): return self.has_attr("minor") @@ -599,7 +599,7 @@ class Member(Containee): names = self.member_type.get_pointer_names(marshalled) if self.has_attr("outvar"): prefix = self.attributes["outvar"][0] - names = map(lambda name: prefix + "_" + name, names) + names = [prefix + "_" + name for name in names] return names class SwitchCase: @@ -656,7 +656,7 @@ class Switch(Containee): def resolve(self, container): self.container = container - self.cases = map(lambda c : c.resolve(self), self.cases) + self.cases = [c.resolve(self) for c in self.cases] return self def __repr__(self): @@ -703,7 +703,7 @@ class Switch(Containee): def get_fixed_nw_size(self): if not self.is_fixed_nw_size(): - raise Exception, "Not a fixed size type" + raise Exception("Not a fixed size type") size = 0 for c in self.cases: size = max(size, c.member.get_fixed_nw_size()) @@ -774,7 +774,7 @@ class ContainerType(Type): return size def resolve(self): - self.members = map(lambda m : m.resolve(self), self.members) + self.members = [m.resolve(self) for m in self.members] return self def get_num_pointers(self): @@ -819,7 +819,7 @@ class ContainerType(Type): name = name[:dot] member = None - if self.members_by_name.has_key(name): + if name in self.members_by_name: member = self.members_by_name[name] else: for m in self.members: @@ -831,7 +831,7 @@ class ContainerType(Type): break if member == None: - raise Exception, "No member called %s found" % name + raise Exception("No member called %s found" % name) if rest != None: return member.member_type.lookup_member(rest) @@ -880,7 +880,7 @@ class MessageType(ContainerType): def c_name(self): if self.name == None: - cms = self.reverse_members.keys() + cms = list(self.reverse_members.keys()) if len(cms) != 1: raise "Unknown typename for message" cm = cms[0] @@ -900,7 +900,7 @@ class MessageType(ContainerType): if self.has_attr("ctype"): return self.attributes["ctype"][0] if self.name == None: - cms = self.reverse_members.keys() + cms = list(self.reverse_members.keys()) if len(cms) != 1: raise "Unknown typename for message" cm = cms[0] diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index 44456ab..d60bb10 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -3,11 +3,12 @@ try: Forward, delimitedList, Group, Optional, Combine, alphas, nums, restOfLine, cStyleComment, \ alphanums, ParseException, ParseResults, Keyword, StringEnd, replaceWith except ImportError: - print "Module pyparsing not found." + six.print_("Module pyparsing not found.") exit(1) -import ptypes +from . import ptypes +import six import sys cvtInt = lambda toks: int(toks[0]) @@ -148,10 +149,10 @@ def parse(filename): try: bnf = SPICE_BNF() types = bnf.parseFile(filename) - except ParseException, err: - print >> sys.stderr, err.line - print >> sys.stderr, " "*(err.column-1) + "^" - print >> sys.stderr, err + except ParseException as err: + six.print_(err.line, file=sys.stderr) + six.print_(" "*(err.column-1) + "^", file=sys.stderr) + six.print_(err, file=sys.stderr) return None for t in types: diff --git a/spice_codegen.py b/spice_codegen.py index c8376cc..16ad478 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -9,6 +9,7 @@ from python_modules import ptypes from python_modules import codegen from python_modules import demarshal from python_modules import marshal +import six def write_channel_enums(writer, channel, client, describe): messages = filter(lambda m : m.channel == channel, \ @@ -257,15 +258,18 @@ if options.keep_identical_file: f.close() if content == old_content: - print "No changes to %s" % dest_file + six.print_("No changes to %s" % dest_file) sys.exit(0) except IOError: pass f = open(dest_file, 'wb') -f.write(content) +if six.PY2: + f.write(content) +else: + f.write(bytes(content, 'UTF-8')) f.close() -print "Wrote %s" % dest_file +six.print_("Wrote %s" % dest_file) sys.exit(0) From 3cc9566a00cb4deb1ead414c390a29beaccc0bd0 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Tue, 14 Apr 2015 16:08:43 +0200 Subject: [PATCH 133/147] codegen: Use six.PY3 rather than six.PY2 Older versions of python-six (at least 1.3.0) defined six.PY3 but not six.PY2. six.PY2 is only used twice in straightforward tests so it's easy to use six.PY3 instead. --- python_modules/codegen.py | 6 +++--- spice_codegen.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 55f513b..f324498 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -121,10 +121,10 @@ class CodeWriter: def write(self, s): # Ensure its a unicode string - if six.PY2: - s = unicode(s) - else: + if six.PY3: s = str(s) + else: + s = unicode(s) if len(s) == 0: return diff --git a/spice_codegen.py b/spice_codegen.py index 16ad478..84790af 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -265,10 +265,10 @@ if options.keep_identical_file: pass f = open(dest_file, 'wb') -if six.PY2: - f.write(content) -else: +if six.PY3: f.write(bytes(content, 'UTF-8')) +else: + f.write(content) f.close() six.print_("Wrote %s" % dest_file) From 12ab27180b3cae25096a1c5281a6f26d14470dc5 Mon Sep 17 00:00:00 2001 From: Javier Celaya Date: Mon, 27 Apr 2015 10:01:00 +0200 Subject: [PATCH 134/147] Proto: Add preferred compression message and constants. When accessing a virtual desktop from different devices, some may have different image compression requirements, e.g. slow devices may prefer the faster LZ4 over GLZ. This message instructs the server to switch the image compression algorithm. This patch also promotes the SPICE_IMAGE_COMPRESS_* constants so that they are available from both the server and the client. --- spice.proto | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spice.proto b/spice.proto index 01493c9..2889802 100644 --- a/spice.proto +++ b/spice.proto @@ -361,6 +361,17 @@ enum8 image_type { LZ4, }; +enum8 image_compress { + INVALID = 0, + OFF, + AUTO_GLZ, + AUTO_LZ, + QUIC, + GLZ, + LZ, + LZ4, +}; + flags8 image_flags { CACHE_ME, HIGH_BITS_SET, @@ -922,6 +933,10 @@ channel DisplayChannel : BaseChannel { int32 last_frame_delay; uint32 audio_delay; } stream_report; + + message { + uint8 image_compression; + } preferred_compression; }; flags16 keyboard_modifier_flags { From 5adcb1c340ce7427991714b0b03f49bb81f502e1 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Mon, 20 Jul 2015 20:55:47 +0200 Subject: [PATCH 135/147] proto: Remove space before tab There are 3 lines in spice.proto/spice1.proto which start with spaces and then contain a tab. This commit removes the spaces and only keep the tab. --- spice.proto | 4 ++-- spice1.proto | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spice.proto b/spice.proto index 2889802..4ea1263 100644 --- a/spice.proto +++ b/spice.proto @@ -185,7 +185,7 @@ channel BaseChannel { Data migrate_data; message { - uint64 time_stamp; + uint64 time_stamp; link_err reason; } @ctype(SpiceMsgDisconnect) disconnecting; }; @@ -887,7 +887,7 @@ channel DisplayChannel : BaseChannel { } monitors_config; message { - DisplayBase base; + DisplayBase base; struct Composite { composite_flags flags; Image *src_bitmap; diff --git a/spice1.proto b/spice1.proto index 67eb0e6..6adf312 100644 --- a/spice1.proto +++ b/spice1.proto @@ -151,7 +151,7 @@ channel BaseChannel { Data migrate_data; message { - uint64 time_stamp; + uint64 time_stamp; link_err reason; } @ctype(SpiceMsgDisconnect) disconnecting; }; From 9625b45edb548e0e4f88d3cfc3b79f30315c8735 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 22 Jul 2015 12:48:55 +0200 Subject: [PATCH 136/147] codegen: Fix enums.h generation with python3 Trying to generate enums.h with python3 results in Traceback (most recent call last): File "./spice_codegen.py", line 217, in write_enums(writer, options.generate_dissector) File "./spice_codegen.py", line 99, in write_enums write_channel_enums(writer, c, False, False) File "./spice_codegen.py", line 17, in write_channel_enums if len(messages) == 0: TypeError: object of type 'filter' has no len() filter() returns an enumerator object in python3 while it used to return a list in python2. Using list(filter()) instead fixes that error. I've checked that the generated enums.h is identical with python2 and python3. --- spice_codegen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spice_codegen.py b/spice_codegen.py index 84790af..569cccc 100755 --- a/spice_codegen.py +++ b/spice_codegen.py @@ -12,8 +12,8 @@ from python_modules import marshal import six def write_channel_enums(writer, channel, client, describe): - messages = filter(lambda m : m.channel == channel, \ - channel.client_messages if client else channel.server_messages) + messages = list(filter(lambda m : m.channel == channel, \ + channel.client_messages if client else channel.server_messages)) if len(messages) == 0: return if client: From 4a2a9966749c09465aa85a5fbd19cf7995d06f45 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:31 +0100 Subject: [PATCH 137/147] codegen: Import six module before first use The module is used in the initial try/except so make sure it is already imported. Signed-off-by: Frediano Ziglio --- python_modules/spice_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index d60bb10..80b559b 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -1,3 +1,5 @@ +import six + try: from pyparsing import Literal, CaselessLiteral, Word, OneOrMore, ZeroOrMore, \ Forward, delimitedList, Group, Optional, Combine, alphas, nums, restOfLine, cStyleComment, \ @@ -8,7 +10,6 @@ except ImportError: from . import ptypes -import six import sys cvtInt = lambda toks: int(toks[0]) From 5bad231e80ad6202b1860783cc29093b0457d296 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:32 +0100 Subject: [PATCH 138/147] codegen: Simplify if/else blocks Blocks were mainly the same, this reduces the amount of code. Signed-off-by: Frediano Ziglio --- python_modules/marshal.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/python_modules/marshal.py b/python_modules/marshal.py index b77b910..1d38d3d 100644 --- a/python_modules/marshal.py +++ b/python_modules/marshal.py @@ -380,25 +380,18 @@ def write_protocol_marshaller(writer, proto, is_server, private_marshallers): writer.ifdef(channel.attributes["ifdef"][0]) writer.header.ifdef(channel.attributes["ifdef"][0]) if is_server: - for m in channel.client_messages: - message = m.message_type - f = write_message_marshaller(writer, message, is_server, private_marshallers) - if channel.has_attr("ifdef") and f not in functions: - functions[f] = channel.attributes["ifdef"][0] - elif message.has_attr("ifdef") and f not in functions: - functions[f] = message.attributes["ifdef"][0] - else: - functions[f] = True + messages = channel.client_messages else: - for m in channel.server_messages: - message = m.message_type - f = write_message_marshaller(writer, message, is_server, private_marshallers) - if channel.has_attr("ifdef") and f not in functions: - functions[f] = channel.attributes["ifdef"][0] - elif message.has_attr("ifdef") and f not in functions: - functions[f] = message.attributes["ifdef"][0] - else: - functions[f] = True + messages = channel.server_messages + for m in messages: + message = m.message_type + f = write_message_marshaller(writer, message, is_server, private_marshallers) + if channel.has_attr("ifdef") and f not in functions: + functions[f] = channel.attributes["ifdef"][0] + elif message.has_attr("ifdef") and f not in functions: + functions[f] = message.attributes["ifdef"][0] + else: + functions[f] = True if channel.has_attr("ifdef"): writer.endif(channel.attributes["ifdef"][0]) writer.header.endif(channel.attributes["ifdef"][0]) From 233c463e3461c5846f125b25c37c6f74b36f1f3c Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:33 +0100 Subject: [PATCH 139/147] codegen: Fix typo in variable name Signed-off-by: Frediano Ziglio --- python_modules/codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index f324498..55500b7 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -193,7 +193,7 @@ class CodeWriter: def unindent(self): self.indentation -= 4 if self.indentation < 0: - self.indenttation = 0 + self.indentation = 0 def begin_block(self, prefix= "", comment = ""): if len(prefix) > 0: From 553be710677bd450b00abb4a95680b506bfcb574 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:34 +0100 Subject: [PATCH 140/147] codegen: Optimize code indentation and avoid a loop Signed-off-by: Frediano Ziglio --- python_modules/codegen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 55500b7..02ffdb9 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -130,8 +130,7 @@ class CodeWriter: return if self.at_line_start: - for i in range(self.indentation): - self.out.write(u" ") + self.out.write(u" " * self.indentation) self.at_line_start = False self.out.write(s) return self From 13aabda256b8a8342735b52fc3fb9e426f581dae Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:35 +0100 Subject: [PATCH 141/147] codegen: Remove duplicate variable initialization Signed-off-by: Frediano Ziglio --- python_modules/spice_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_modules/spice_parser.py b/python_modules/spice_parser.py index 80b559b..97af8b2 100644 --- a/python_modules/spice_parser.py +++ b/python_modules/spice_parser.py @@ -58,7 +58,6 @@ def SPICE_BNF(): uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64)) # keywords - channel_ = Keyword("channel") enum32_ = Keyword("enum32").setParseAction(replaceWith(32)) enum16_ = Keyword("enum16").setParseAction(replaceWith(16)) enum8_ = Keyword("enum8").setParseAction(replaceWith(8)) From 6f729cb32ccda351451f1cd0c9eac031fc007986 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:36 +0100 Subject: [PATCH 142/147] codegen: Reuse code to fix attribute from prototype file Signed-off-by: Frediano Ziglio --- python_modules/ptypes.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index d031d09..845fa73 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -62,6 +62,14 @@ class FixedSize: # other members propagated_attributes=["ptr_array", "nonnull", "chunk"] +def fix_attributes(attribute_list): + attrs = {} + for attr in attribute_list: + name = attr[0][1:] + lst = attr[1:] + attrs[name] = lst + return attrs + class Type: def __init__(self): self.attributes = {} @@ -178,8 +186,7 @@ class TypeAlias(Type): Type.__init__(self) self.name = name self.the_type = the_type - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def get_type(self, recursive=False): if recursive: @@ -288,8 +295,7 @@ class EnumType(EnumBaseType): self.names = names self.values = values - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def __str__(self): return "enum %s" % self.name @@ -342,8 +348,7 @@ class FlagsType(EnumBaseType): self.names = names self.values = values - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def __str__(self): return "flags %s" % self.name @@ -533,8 +538,7 @@ class Member(Containee): Containee.__init__(self) self.name = name self.member_type = member_type - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def resolve(self, container): self.container = container @@ -636,8 +640,7 @@ class Switch(Containee): self.variable = variable self.name = name self.cases = cases - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def is_switch(self): return True @@ -846,8 +849,7 @@ class StructType(ContainerType): self.members_by_name = {} for m in members: self.members_by_name[m.name] = m - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def __str__(self): if self.name == None: @@ -869,8 +871,7 @@ class MessageType(ContainerType): for m in members: self.members_by_name[m.name] = m self.reverse_members = {} # ChannelMembers referencing this message - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def __str__(self): if self.name == None: @@ -938,8 +939,7 @@ class ChannelType(Type): self.base = base self.member_name = None self.members = members - for attr in attribute_list: - self.attributes[attr[0][1:]] = attr[1:] + self.attributes = fix_attributes(attribute_list) def __str__(self): if self.name == None: From 25c48ddd2ceb4b6a46ebafc7dabbb4973ad06ffe Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:37 +0100 Subject: [PATCH 143/147] codegen: Do some checks on attributes Verify that the attribute is known. This could help for instance to avoid some future typo mistakes. We also now have a list of attributes that we can comment for documentation purpose. Signed-off-by: Frediano Ziglio --- python_modules/ptypes.py | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 845fa73..5cd7759 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -62,11 +62,79 @@ class FixedSize: # other members propagated_attributes=["ptr_array", "nonnull", "chunk"] +valid_attributes={ + # embedded/appended at the end of the structure + 'end', + # the C structure contains a pointer to data + # for instance we want to write an array to an allocated array + 'to_ptr', + # write output to this C structure + 'ctype', + # prefix for flags/values enumerations + 'prefix', + # used in demarshaller to use directly data from message without a copy + 'nocopy', + # store member array in a pointer + # similar to to_ptr but has an additional argument which is the name of a C + # field which will store the array length + 'as_ptr', + # do not generate marshall code + # used for last members to be able to marshall them manually + 'nomarshal', + # ??? not used by python code + 'zero_terminated', + 'marshall', + # this pointer member cannot be null + 'nonnull', + # this flag member contains only a single flag + 'unique_flag', + 'ptr_array', + 'outvar', + # C structure has an anonymous member (used in switch) + 'anon', + 'chunk', + # this channel is contained in an #ifdef section + # the argument specifies the preprocessor define to check + 'ifdef', + # write this member as zero on network + 'zero', + # specify minor version required for these members + 'minor', + # this member contains the byte count for an array. + # the argument is the member name for item count (not bytes) + 'bytes_count', + # this attribute does not exist on the network, fill just structure with the value + 'virtual', + # for a switch this indicates that on network + # it will occupy always the same size (maximum size required for all members) + 'fixedsize', + # use 32 bit pointer + 'ptr32', +} + +attributes_with_arguments={ + 'ctype', + 'prefix', + 'as_ptr', + 'outvar', + 'ifdef', + 'minor', + 'bytes_count', + 'virtual', +} + def fix_attributes(attribute_list): attrs = {} for attr in attribute_list: name = attr[0][1:] lst = attr[1:] + if not name in valid_attributes: + raise Exception("Attribute %s not recognized" % name) + if not name in attributes_with_arguments: + if len(lst) > 0: + raise Exception("Attribute %s specified with options" % name) + elif len(lst) > 1: + raise Exception("Attribute %s has more than 1 argument" % name) attrs[name] = lst return attrs @@ -139,6 +207,8 @@ class Type: _types_by_name[self.name] = self def has_attr(self, name): + if not name in valid_attributes: + raise Exception('attribute %s not expected' % name) return name in self.attributes class TypeRef(Type): @@ -522,6 +592,8 @@ class Containee: return not self.is_switch() and self.member_type.is_primitive() def has_attr(self, name): + if not name in valid_attributes: + raise Exception('attribute %s not expected' % name) return name in self.attributes def has_minor_attr(self): From f6506a25639a47cfdcef452202c1ee9233be3f36 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:38 +0100 Subject: [PATCH 144/147] codegen: Remove old ptr32 attribute This attribute is not used in code. Signed-off-by: Frediano Ziglio --- python_modules/ptypes.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index 5cd7759..efbe9b6 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -108,8 +108,6 @@ valid_attributes={ # for a switch this indicates that on network # it will occupy always the same size (maximum size required for all members) 'fixedsize', - # use 32 bit pointer - 'ptr32', } attributes_with_arguments={ @@ -616,8 +614,6 @@ class Member(Containee): self.container = container self.member_type = self.member_type.resolve() self.member_type.register() - if self.has_attr("ptr32") and self.member_type.is_pointer(): - self.member_type.set_ptr_size(4) for i in propagated_attributes: if self.has_attr(i): self.member_type.attributes[i] = self.attributes[i] From 08384ac7fbdc8303fdf85cc2689602e429607126 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:39 +0100 Subject: [PATCH 145/147] codegen: Check we don't pop too many indexes --- python_modules/codegen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index 02ffdb9..c470988 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -357,6 +357,7 @@ class CodeWriter: return index def push_index(self): + assert self.current_index > 0 self.current_index = self.current_index - 1 class Index: From de1286ad80ce7c559fccde5f6e466a89fc1ce60d Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Tue, 21 Jul 2015 17:45:40 +0100 Subject: [PATCH 146/147] codegen: Allow to specify C type for index variable This is to prepare to generate the wireshark dissector which uses glib types instead of the newer C ones (for compatibility with some compilers). Signed-off-by: Frediano Ziglio --- python_modules/codegen.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python_modules/codegen.py b/python_modules/codegen.py index c470988..f7a2048 100644 --- a/python_modules/codegen.py +++ b/python_modules/codegen.py @@ -81,6 +81,7 @@ class CodeWriter: self.has_error_check = False self.options = {} self.function_helper_writer = None + self.index_type = 'uint32_t' def set_option(self, opt, value = True): self.options[opt] = value @@ -113,6 +114,7 @@ class CodeWriter: self.contents.append(self.out) writer.indentation = self.indentation writer.at_line_start = self.at_line_start + writer.index_type = self.index_type writer.generated = self.generated writer.options = self.options writer.public_prefix = self.public_prefix @@ -353,7 +355,7 @@ class CodeWriter: def pop_index(self): index = self.indexes[self.current_index] self.current_index = self.current_index + 1 - self.add_function_variable("uint32_t", index) + self.add_function_variable(self.index_type, index) return index def push_index(self): From 1d026b9a71a12df119f5582c9a42be08259b0bac Mon Sep 17 00:00:00 2001 From: Uri Lublin Date: Mon, 27 Jul 2015 20:14:26 +0300 Subject: [PATCH 147/147] codegen: ptypes.py: keep attribute names in sets This patch changes the type of 'valid_attributes' and 'attributes_with_arguments'. Both of them are list of different strings and are kept in sets. This was the intention of the original code, but this patch use a specific set([ strings ]) format, instead of { strings }. This fixes the build for me on RHEL-6 (python-2.6.6). Build error is: File "/home/ulublin/git/spice/spice-common/python_modules/ptypes.py", line 67 'end', ^ SyntaxError: invalid syntax --- python_modules/ptypes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python_modules/ptypes.py b/python_modules/ptypes.py index efbe9b6..7ab2771 100644 --- a/python_modules/ptypes.py +++ b/python_modules/ptypes.py @@ -62,7 +62,7 @@ class FixedSize: # other members propagated_attributes=["ptr_array", "nonnull", "chunk"] -valid_attributes={ +valid_attributes=set([ # embedded/appended at the end of the structure 'end', # the C structure contains a pointer to data @@ -108,9 +108,9 @@ valid_attributes={ # for a switch this indicates that on network # it will occupy always the same size (maximum size required for all members) 'fixedsize', -} +]) -attributes_with_arguments={ +attributes_with_arguments=set([ 'ctype', 'prefix', 'as_ptr', @@ -119,7 +119,7 @@ attributes_with_arguments={ 'minor', 'bytes_count', 'virtual', -} +]) def fix_attributes(attribute_list): attrs = {}