proto : impossible d'analyser les données de format de fil invalides

Contenu de la question

Je suis nouveau sur protobufs et j'écris actuellement un client qui lit les données d'un serveur nats. Les données envoyées depuis le serveur Nats sont protobuf.

Le client que j'écris est écrit en go. Voici le fichier .proto que j'ai écrit :

syntax = "proto3";

package execution;

option go_package = "./protos/execution";

enum orderstatus {
  working = 0;
  rejected = 1;
  cancelled = 2;
  completed = 3;
}

enum ordertype {
  limit = 0;
  market = 1;
  stoplimit = 2;
  stopmarket = 3;
}

enum orderside {
  buy = 0;
  sell = 1;
}

enum rejectreason {
  norejection = 0;
  instrumentnotfound = 1;
  ordernotfound = 2;
  invalidordertype = 3;
  invalidaccount = 4;
  invalidside = 5;
  invalidamount = 6;
  invalidlimitprice = 7;
  invalidquotelimit = 8;
  invalidactivationprice = 9;
  invalidtimeinforce = 10;
  markethalted = 11;
  marketpaused = 12;
  nocounterorders = 13;
  missingexpirationtime = 14;
  incorrectexpirationtime = 15;
  internalerror = 16;
  illegalstatusswitch = 17;
  orderalreadyexists = 18;
  instrumentnotready = 19;
  externalsystemerror = 20;
}

enum reportcause {
  none = 0;
  neworder = 1;
  cancelorder = 2;
  masscancel = 3;
  expiration = 4;
  trigger = 5;
  marketstatuschange = 6;
}

enum timeinforce {
  goodtillcancel = 0;
  immediateorcancel = 1;
  fillorkill = 2;
}

enum cancelreason {
  notcancelled = 0;
  cancelledbytrader = 1;
  cancelledbysystem = 2;
  selfmatchprevention = 3;
  ordertimeinforce = 4;
  liquidation = 100;
}


message tradedata {
  int64 tradeid = 1;
  string amount = 4;
  string executionprice = 5;
  orderstatus orderstatus = 7;
  int64 accountid = 11;
  string matchedorderexternalid = 14;
  int64 matchedorderid = 16;
  string remainingamount = 17;
}

message execution {
  string origin = 4;
  orderside side = 7;
  string requestedprice = 8;
  string requestedamount = 9;
  string remainingamount = 10;
  int64 executedat = 13;
  orderstatus orderstatus = 14;
  repeated tradedata trades = 16;
  ordertype ordertype = 20;
  int64 version = 22;
  int64 accountid = 23;
  rejectreason rejectreason = 25;
  reportcause reportcause = 26;
  string instructionid = 27;
  string externalorderid = 28;
  int32 executionenginemarketid = 29;
  int64 orderid = 30;
  cancelreason cancelreason = 31;
  int64 txid = 32;
  timeinforce timeinforce = 34;
  string cancelledby = 35;
}

Le serveur de publication est écrit en c# et le code de son message d'origine est le suivant :

[protocontract]
    public class executionreport : imarketresponse, iinstructionmessage, iordermatcherresponse
    {
        [protoignore]
        feedmessagetype ifeedmessage.type => feedmessagetype.executionreport;

        // resharper disable fieldcanbemadereadonly.global
        [protomember(4)] public string origin;
        [protomember(7)] public orderside side;
        [protomember(8)] public decimal requestedprice;
        [protomember(9)] public decimal requestedamount;
        [protomember(10)] public decimal remainingamount;
        [protomember(13)] public long executedat;
        [protomember(14)] public orderstatus orderstatus;
        [protomember(16)] public list<tradedata> trades = new list<tradedata>();
        [protomember(20)] public ordertype ordertype;
        [protomember(22)] public long version { get; set; }
        [protomember(23)] public long accountid;
        [protomember(25)] public rejectreason rejectreason;
        [protomember(26)] public reportcause reportcause;
        [protomember(27)] public guid instructionid { get; set; }
        [protomember(28)] public guid externalorderid;
        [protomember(29)] public int executionenginemarketid { get; set; }
        [protomember(30)] public long orderid;
        [protomember(31)] public cancelreason cancelreason;
        [protomember(32)] public long txid;
        [protomember(34)] public timeinforce timeinforce;
        [protomember(35)] public string cancelledby;
    }

[protocontract]
    [structlayout(layoutkind.sequential)]
    public struct tradedata
    {
        [protomember(1)] public long tradeid;
        [protomember(4)] public decimal amount;
        [protomember(5)] public decimal executionprice;
        [protomember(7)] public orderstatus orderstatus;
        [protomember(11)] public long accountid;
        [protomember(14)] public guid matchedorderexternalid;
        [protomember(16)] public long matchedorderid;
        [protomember(17)] public decimal remainingamount;
    }

J'obtiens cette erreur lorsque j'essaie de désorganiser les données

proto: cannot parse invalid wire-format data

Voici comment j'analyse les données :

_, err = sc.subscribe("exec", func(m *stan.msg) {
varr := &protos.execution{}
err = proto.unmarshal(m.data, varr)
if err != nil {
    fmt.printf("err unmarshalling!: %v\n\n", err.error())
} else {
    fmt.printf("received a message: %+v\n", varr)
}

Exemples de données d'octets que j'ai reçues du serveur :

[5 85 0 0 0 56 1 66 3 8 144 78 74 2 8 1 82 2 8 1 104 197 192 132 194 159 143 219 237 8 176 1 25 184 1 11 208 1 1 218 1 18 9 133 66 138 247 239 67 93 77 17 176 192 189 75 170 203 186 145 226 1 18 9 133 66 138 247 239 67 93 77 17 176 192 189 75 170 203 186 145 232 1 1 240 1 25 128 2 25]

Ajoutez plus de détails :

Voici comment c# envoie les données :

public async task sendasync(ifeedmessage msg)
{
    var subject = feedsubject.formessage(msg);
    var data = msg.serializetoarray();
    using (_metrics.feedsendlatency.start(new metrictags("subject", subject.value)))
    {
        await _connection.publishasync(subject, data);
    }
}

Il s'agit de la structure de feedmessage (executionreport en hérite également indirectement)

public interface ifeedmessage
{
    feedmessagetype type { get; }
    ifeedmessage clone();
    void reset();
}

Voici comment serializetoarray() fonctionne :

public static ArraySegment<byte> SerializeToArray(this IFeedMessage message)
{
    return message.SerializeToMemory(new MemoryStream());
}

public static ArraySegment<byte> SerializeToMemory(this IFeedMessage message, MemoryStream stream)
{
    var start = stream.Position;
    message.Serialize(stream);
    return new ArraySegment<byte>(stream.GetBuffer(), (int)start, (int)(stream.Position - start));
}

public static void Serialize(this IFeedMessage message, Stream stream)
{
    stream.WriteByte((byte)message.Type);
    RuntimeTypeModel.Default.SerializeWithLengthPrefix(stream, message, message.GetType(), PrefixStyle.Fixed32, 0);
}

Je ne connais pas la raison précise. Mais le fichier proto que j'ai écrit semble être erroné. J'ai parcouru plusieurs messages confrontés à la même erreur, mais la plupart d'entre eux n'ont pas résolu le même problème. Si d'autres détails sont nécessaires, veuillez me le faire savoir.

S'il vous plaît, aidez-moi à résoudre ce problème.

Bonne réponse

Sur la base de la discussion dans les commentaires, j'ai réussi à organiser les données.

Remarques :

1. Les données sont préfixées par 5 octets (c'est complètement inutile) :
   • Type de message 1 octet
   • La longueur des données est de 4 octets
2. Les implémentations c# utilisent des decimal 和 guid 数据类型。 （正如 bcl.proto spécifiques à c# comme commenté dans
, le code multiplateforme devrait généralement les éviter complètement).

Voici la structure des dossiers :

├── bcl.proto
├── execution.proto
├── go.mod
├── go.sum
├── main.go
└── protos
    ├── bcl.pb.go
    └── execution.pb.go

bcl.proto

:

Ce fichier a été copié depuis github. com/protobuf-net/protobuf-netdecimal 和 guid. Ceci est nécessaire car l'implémentation .net utilise le

dans ce fichier original.

// the types in here indicate how protobuf-net represents certain types when using protobuf-net specific
// library features. note that it is not *required* to use any of these types, and cross-platform code
// should usually avoid them completely (ideally starting from a .proto schema)

// some of these are ugly, sorry. the timespan / datetime dates here pre-date the introduction of timestamp
// and duration, and the "well known" types should be preferred when possible. guids are particularly
// awkward - it turns out that there are multiple guid representations, and i accidentally used one that
// i can only call... "crazy-endian". just make sure you check the order!

// it should not be necessary to use bcl.proto from code that uses protobuf-net

syntax = "proto3";

option csharp_namespace = "protobuf.bcl";
option go_package = "./protos";

package bcl;

message timespan {
  sint64 value = 1; // the size of the timespan (in units of the selected scale)
  timespanscale scale = 2; // the scale of the timespan [default = days]
  enum timespanscale {
    days = 0;
    hours = 1;
    minutes = 2;
    seconds = 3;
    milliseconds = 4;
    ticks = 5;

    minmax = 15; // dubious
  }
}

message datetime {
  sint64 value = 1; // the offset (in units of the selected scale) from 1970/01/01
  timespanscale scale = 2; // the scale of the timespan [default = days]
  datetimekind kind = 3; // the kind of date/time being represented [default = unspecified]
  enum timespanscale {
    days = 0;
    hours = 1;
    minutes = 2;
    seconds = 3;
    milliseconds = 4;
    ticks = 5;

    minmax = 15; // dubious
  }
  enum datetimekind
  {
     // the time represented is not specified as either local time or coordinated universal time (utc).
     unspecified = 0;
     // the time represented is utc.
     utc = 1;
     // the time represented is local time.
     local = 2;
   }
}

message netobjectproxy {
  int32 existingobjectkey = 1; // for a tracked object, the key of the **first** time this object was seen
  int32 newobjectkey = 2; // for a tracked object, a **new** key, the first time this object is seen
  int32 existingtypekey = 3; // for dynamic typing, the key of the **first** time this type was seen
  int32 newtypekey = 4; // for dynamic typing, a **new** key, the first time this type is seen
  string typename = 8; // for dynamic typing, the name of the type (only present along with newtypekey)
  bytes payload = 10; // the new string/value (only present along with newobjectkey)
}

message guid {
  fixed64 lo = 1; // the first 8 bytes of the guid (note:crazy-endian)
  fixed64 hi = 2; // the second 8 bytes of the guid (note:crazy-endian)
}

message decimal {
  uint64 lo = 1; // the first 64 bits of the underlying value
  uint32 hi = 2; // the last 32 bis of the underlying value
  uint32 signscale = 3; // the number of decimal digits (bits 1-16), and the sign (bit 0)
}

execution.proto

syntax = "proto3";

package execution;

option go_package = "./protos";

import "bcl.proto";

enum orderstatus {
  working = 0;
  rejected = 1;
  cancelled = 2;
  completed = 3;
}

enum ordertype {
  limit = 0;
  market = 1;
  stoplimit = 2;
  stopmarket = 3;
}

enum orderside {
  buy = 0;
  sell = 1;
}

enum rejectreason {
  norejection = 0;
  instrumentnotfound = 1;
  ordernotfound = 2;
  invalidordertype = 3;
  invalidaccount = 4;
  invalidside = 5;
  invalidamount = 6;
  invalidlimitprice = 7;
  invalidquotelimit = 8;
  invalidactivationprice = 9;
  invalidtimeinforce = 10;
  markethalted = 11;
  marketpaused = 12;
  nocounterorders = 13;
  missingexpirationtime = 14;
  incorrectexpirationtime = 15;
  internalerror = 16;
  illegalstatusswitch = 17;
  orderalreadyexists = 18;
  instrumentnotready = 19;
  externalsystemerror = 20;
}

enum reportcause {
  none = 0;
  neworder = 1;
  cancelorder = 2;
  masscancel = 3;
  expiration = 4;
  trigger = 5;
  marketstatuschange = 6;
}

enum timeinforce {
  goodtillcancel = 0;
  immediateorcancel = 1;
  fillorkill = 2;
}

enum cancelreason {
  notcancelled = 0;
  cancelledbytrader = 1;
  cancelledbysystem = 2;
  selfmatchprevention = 3;
  ordertimeinforce = 4;
  liquidation = 100;
}


message tradedata {
  int64 tradeid = 1;
  bcl.decimal amount = 4;
  bcl.decimal executionprice = 5;
  orderstatus orderstatus = 7;
  int64 accountid = 11;
  bcl.guid matchedorderexternalid = 14;
  int64 matchedorderid = 16;
  bcl.decimal remainingamount = 17;
}

message execution {
  bytes origin = 4;
  orderside side = 7;
  bcl.decimal requestedprice = 8;
  bcl.decimal requestedamount = 9;
  bcl.decimal remainingamount = 10;
  int64 executedat = 13;
  orderstatus orderstatus = 14;
  repeated tradedata trades = 16;
  ordertype ordertype = 20;
  int64 version = 22;
  int64 accountid = 23;
  rejectreason rejectreason = 25;
  reportcause reportcause = 26;
  bcl.guid instructionid = 27;
  bcl.guid externalorderid = 28;
  int32 executionenginemarketid = 29;
  int64 orderid = 30;
  cancelreason cancelreason = 31;
  int64 txid = 32;
  timeinforce timeinforce = 34;
  string cancelledby = 35;
}

Prototype/

Les fichiers de ce dossier sont générés à partir de fichiers proto à l'aide de la commande suivante :

protoc --go_out=protos --go_opt=paths=source_relative bcl.proto execution.proto

go.mod

module mymodule.local

go 1.20

require google.golang.org/protobuf v1.30.0

main.go

package main

import (
    "encoding/binary"
    "log"

    "google.golang.org/protobuf/proto"

    "mymodule.local/protos"
)

func main() {
    data := []byte{5, 85, 0, 0, 0, 56, 1, 66, 3, 8, 144, 78, 74, 2, 8, 1, 82, 2, 8, 1, 104, 197, 192, 132, 194, 159, 143, 219, 237, 8, 176, 1, 25, 184, 1, 11, 208, 1, 1, 218, 1, 18, 9, 133, 66, 138, 247, 239, 67, 93, 77, 17, 176, 192, 189, 75, 170, 203, 186, 145, 226, 1, 18, 9, 133, 66, 138, 247, 239, 67, 93, 77, 17, 176, 192, 189, 75, 170, 203, 186, 145, 232, 1, 1, 240, 1, 25, 128, 2, 25}
    if len(data) < 5 {
        log.fatal("data should contain at least 5 bytes")
    }
    messagetype := data[0]
    length := binary.littleendian.uint32(data[1:5])
    data = data[5:]
    if length != uint32(len(data)) {
        log.fatalf("invalid data length: %d", length)
    }
    execution := &protos.execution{}

    err := proto.unmarshal(data, execution)
    if err != nil {
        log.fatalf("err unmarshalling!: %v", err)
    }

    log.printf("message type: %d, message: %+v", messagetype, execution)
}

Sortie des données fournies dans la question : 🎜

2023/06/15 17:50:58 message type: 5, message: Side:Sell  RequestedPrice:{lo:10000}  RequestedAmount:{lo:1}  RemainingAmount:{lo:1}  ExecutedAt:638223043314917445  Version:25  AccountId:11  ReportCause:NewOrder  InstructionId:{lo:5574686611683820165  hi:10500929413443338416}  ExternalOrderId:{lo:5574686611683820165  hi:10500929413443338416}  ExecutionEngineMarketId:1  OrderId:25  TxId:25

Ce qui précède est le contenu détaillé de. pour plus d'informations, suivez d'autres articles connexes sur le site Web de PHP en chinois!