From 402291008ea3e392716a935b43a5ee0116d399f9 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Sat, 30 Sep 2023 13:14:40 -0300 Subject: [PATCH] upgrade .SerializeInstructionSequences() to the crazy checkpointing scheme. --- helpers.go | 52 +++++++++++++++++++++++++ ots.go | 109 ++++++++++++++++++++++++++++------------------------- parsers.go | 16 ++++---- stamp.go | 15 +++----- utils.go | 12 +++++- 5 files changed, 134 insertions(+), 70 deletions(-) create mode 100644 helpers.go diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..ad8ef05 --- /dev/null +++ b/helpers.go @@ -0,0 +1,52 @@ +package opentimestamps + +import ( + "strings" + + "golang.org/x/exp/slices" +) + +// CompareInstructions returns negative if ab. +// It considers an operation smaller than an attestation, a pending attestation smaller than a Bitcoin attestation. +// It orders operations by their tag byte and then by their argument. +func CompareInstructions(a, b Instruction) int { + if a.Operation != nil { + if b.Attestation != nil { + // a is an operation but b is an attestation, b is bigger + return -1 + } + if a.Operation == b.Operation { + // if both are the same operation sort by the argument + return slices.Compare(a.Argument, b.Argument) + } + + // sort by the operation + if a.Operation.Tag < b.Operation.Tag { + return -1 + } else if a.Operation.Tag > b.Operation.Tag { + return 1 + } else { + return 0 + } + } else if a.Attestation != nil && b.Attestation == nil { + // a is an attestation but b is not, a is bigger + return 1 + } else if a.Attestation != nil && b.Attestation != nil { + // both are attestations + if a.Attestation.BitcoinBlockHeight == 0 && b.Attestation.BitcoinBlockHeight == 0 { + // none are bitcoin attestations + return strings.Compare(a.Attestation.CalendarServerURL, b.Attestation.CalendarServerURL) + } + if a.Attestation.BitcoinBlockHeight != 0 && b.Attestation.BitcoinBlockHeight != 0 { + // both are bitcoin attestations + return int(b.Attestation.BitcoinBlockHeight - a.Attestation.BitcoinBlockHeight) + } + + // one is bitcoin and the other is not -- compare by bitcoin block, + // but reverse the result since the one with 0 should not be considered bigger + return -1 * int(b.Attestation.BitcoinBlockHeight-a.Attestation.BitcoinBlockHeight) + } else { + // this shouldn't happen + return 0 + } +} diff --git a/ots.go b/ots.go index d6c1591..714933d 100644 --- a/ots.go +++ b/ots.go @@ -52,10 +52,10 @@ var tags = map[byte]*Operation{ 0x67: {"keccak256", 0x67, false, func(curr []byte, arg []byte) []byte { panic("keccak256 not implemented") }}, } -// A Timestamp is basically the content of an .ots file: it has an initial digest and +// A File represents the parsed content of an .ots file: it has an initial digest and // a series of sequences of instructions. Each sequence must be evaluated separately, applying the operations // on top of each other, starting with the .Digest until they end on an attestation. -type Timestamp struct { +type File struct { Digest []byte Instructions []Sequence } @@ -69,35 +69,6 @@ type Instruction struct { *Attestation } -func (a Instruction) Equal(b Instruction) bool { - if a.Operation != nil { - if a.Operation == b.Operation && slices.Equal(a.Argument, b.Argument) { - return true - } else { - return false - } - } else if a.Attestation != nil { - if b.Attestation == nil { - return false - } - if a.Attestation.BitcoinBlockHeight != 0 && - a.Attestation.BitcoinBlockHeight == b.Attestation.BitcoinBlockHeight { - return true - } - if a.Attestation.CalendarServerURL != "" && - a.Attestation.CalendarServerURL == b.Attestation.CalendarServerURL { - return true - } - return false - } else { - // a is nil -- this is already broken but whatever - if b.Attestation == nil && b.Operation == nil { - return true - } - return false - } -} - type Sequence []Instruction func (seq Sequence) Compute(initial []byte) []byte { @@ -111,7 +82,7 @@ func (seq Sequence) Compute(initial []byte) []byte { return current } -func (ts Timestamp) GetPendingSequences() []Sequence { +func (ts File) GetPendingSequences() []Sequence { bitcoin := ts.GetBitcoinAttestedSequences() results := make([]Sequence, 0, len(ts.Instructions)) @@ -125,7 +96,7 @@ func (ts Timestamp) GetPendingSequences() []Sequence { continue } - if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return a.Equal(b) }) { + if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return CompareInstructions(a, b) == 0 }) { goto thisSequenceIsAlreadyConfirmed } } @@ -141,7 +112,7 @@ func (ts Timestamp) GetPendingSequences() []Sequence { return results } -func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence { +func (ts File) GetBitcoinAttestedSequences() []Sequence { results := make([]Sequence, 0, len(ts.Instructions)) for _, seq := range ts.Instructions { if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.BitcoinBlockHeight > 0 { @@ -151,7 +122,7 @@ func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence { return results } -func (ts Timestamp) Human() string { +func (ts File) Human() string { strs := make([]string, 0, 100) strs = append(strs, fmt.Sprintf("file digest: %x", ts.Digest)) strs = append(strs, fmt.Sprintf("hashed with: sha256")) @@ -176,7 +147,7 @@ func (ts Timestamp) Human() string { return strings.Join(strs, "\n") } -func (ts Timestamp) SerializeToFile() []byte { +func (ts File) SerializeToFile() []byte { data := make([]byte, 0, 5050) data = append(data, headerMagic...) data = appendVarUint(data, 1) @@ -186,43 +157,77 @@ func (ts Timestamp) SerializeToFile() []byte { return data } -func (ts Timestamp) SerializeInstructionSequences() []byte { - data := make([]byte, 0, 5000) - for i, seq := range ts.Instructions { - for _, inst := range seq { +func (ts File) SerializeInstructionSequences() []byte { + sequences := make([]Sequence, len(ts.Instructions)) + copy(sequences, ts.Instructions) + + // first we sort everything so the checkpoint stuff makes sense + slices.SortFunc(sequences, func(a, b Sequence) int { return slices.CompareFunc(a, b, CompareInstructions) }) + + // checkpoints we may leave to the next people + sequenceCheckpoints := make([][]int, len(sequences)) + for s1 := range sequences { + // keep an ordered slice of all the checkpoints we will potentially leave during our write journey for this sequence + checkpoints := make([]int, 0, len(sequences[s1])) + for s2 := s1 + 1; s2 < len(sequences); s2++ { + chp := getCommonPrefixIndex(sequences[s1], sequences[s2]) + if pos, found := slices.BinarySearch(checkpoints, chp); !found { + checkpoints = append(checkpoints, -1) // make room + copy(checkpoints[pos+1:], checkpoints[pos:]) // move elements to the right + checkpoints[pos] = chp // insert this + } + } + sequenceCheckpoints[s1] = checkpoints + } + + // now actually go through the sequences writing them + result := make([]byte, 0, 500) + for s, seq := range sequences { + startingAt := 0 + if s > 0 { + // we will always start at the last checkpoint left by the previous sequence + startingAt = sequenceCheckpoints[s-1][len(sequenceCheckpoints[s-1])-1] + } + + for i := startingAt; i < len(seq); i++ { + // before writing anything, decide if we wanna leave a checkpoint here + for _, chk := range sequenceCheckpoints[s] { + if chk == i { + // leave a checkpoint + result = append(result, 0xff) + } + } + + inst := seq[i] if inst.Operation != nil { // write normal operation - data = append(data, inst.Operation.Tag) + result = append(result, inst.Operation.Tag) if inst.Operation.Binary { - data = appendVarBytes(data, inst.Argument) + result = appendVarBytes(result, inst.Argument) } } else if inst.Attestation != nil { // write attestation record - data = append(data, 0x00) + result = append(result, 0x00) { - // will use a new buffer for the actual attestation data + // will use a new buffer for the actual attestation result abuf := make([]byte, 0, 100) if inst.BitcoinBlockHeight != 0 { - data = append(data, bitcoinMagic...) // this goes in the main data buffer + result = append(result, bitcoinMagic...) // this goes in the main result buffer abuf = appendVarUint(abuf, inst.BitcoinBlockHeight) } else if inst.CalendarServerURL != "" { - data = append(data, pendingMagic...) // this goes in the main data buffer + result = append(result, pendingMagic...) // this goes in the main result buffer abuf = appendVarBytes(abuf, []byte(inst.CalendarServerURL)) } else { panic(fmt.Sprintf("invalid attestation: %v", inst)) } - data = appendVarBytes(data, abuf) // we append that data as varbytes + result = appendVarBytes(result, abuf) // we append that result as varbytes } } else { panic(fmt.Sprintf("invalid instruction: %v", inst)) } } - if i+1 < len(ts.Instructions) { - // write separator and start a new sequence of instructions - data = append(data, 0xff) - } } - return data + return result } type Attestation struct { diff --git a/parsers.go b/parsers.go index 86ca922..31e8900 100644 --- a/parsers.go +++ b/parsers.go @@ -19,7 +19,7 @@ func parseCalendarServerResponse(buf Buffer) (Sequence, error) { return seqs[0], nil } -func parseOTSFile(buf Buffer) (*Timestamp, error) { +func parseOTSFile(buf Buffer) (*File, error) { // read magic // read version [1 byte] // read crypto operation for file digest [1 byte] @@ -47,7 +47,7 @@ func parseOTSFile(buf Buffer) (*Timestamp, error) { return nil, fmt.Errorf("failed to read 32-byte digest: %w", err) } - ts := &Timestamp{ + ts := &File{ Digest: digest, } @@ -102,7 +102,7 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) { if err != nil { return nil, fmt.Errorf("failed to read attestation bytes: %w", err) } - abuf := NewBuffer(this) + abuf := newBuffer(this) switch { case slices.Equal(magic, pendingMagic): @@ -131,17 +131,17 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) { ncheckpoints := len(checkpoints) if ncheckpoints > 0 { // use this checkpoint as the starting point for the next block - cp := checkpoints[ncheckpoints-1] + chp := checkpoints[ncheckpoints-1] checkpoints = checkpoints[0 : ncheckpoints-1] // remove this from the stack - seqs = append(seqs, cp) + seqs = append(seqs, chp) currInstructionsBlock++ } } else if tag == 0xff { // pick up a checkpoint to be used later currentBlock := seqs[currInstructionsBlock] - cp := make([]Instruction, len(currentBlock)) - copy(cp, currentBlock) - checkpoints = append(checkpoints, cp) + chp := make([]Instruction, len(currentBlock)) + copy(chp, currentBlock) + checkpoints = append(checkpoints, chp) } else { // a new operation in this block inst, err := readInstruction(buf, tag) diff --git a/stamp.go b/stamp.go index 037f5c4..2c2971d 100644 --- a/stamp.go +++ b/stamp.go @@ -8,7 +8,7 @@ import ( "net/http" ) -func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp, error) { +func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (Sequence, error) { body := bytes.NewBuffer(digest[:]) req, err := http.NewRequestWithContext(ctx, "POST", normalizeUrl(calendarUrl)+"/digest", body) if err != nil { @@ -29,19 +29,16 @@ func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp } resp.Body.Close() - seq, err := parseCalendarServerResponse(NewBuffer(full)) + seq, err := parseCalendarServerResponse(newBuffer(full)) if err != nil { return nil, fmt.Errorf("failed to parse response from '%s': %w", calendarUrl, err) } - return &Timestamp{ - Digest: digest[:], - Instructions: []Sequence{seq}, - }, nil + return seq, nil } -func ReadFromFile(data []byte) (*Timestamp, error) { - return parseOTSFile(NewBuffer(data)) +func ReadFromFile(data []byte) (*File, error) { + return parseOTSFile(newBuffer(data)) } func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, error) { @@ -72,7 +69,7 @@ func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, erro } resp.Body.Close() - newSeq, err := parseCalendarServerResponse(NewBuffer(body)) + newSeq, err := parseCalendarServerResponse(newBuffer(body)) if err != nil { return nil, fmt.Errorf("failed to parse response from '%s': %w", attestation.CalendarServerURL, err) } diff --git a/utils.go b/utils.go index 1b99c08..16744cf 100644 --- a/utils.go +++ b/utils.go @@ -20,7 +20,7 @@ type Buffer struct { buf []byte } -func NewBuffer(buf []byte) Buffer { +func newBuffer(buf []byte) Buffer { zero := 0 return Buffer{&zero, buf} } @@ -99,3 +99,13 @@ func appendVarBytes(buf []byte, value []byte) []byte { buf = append(buf, value...) return buf } + +func getCommonPrefixIndex(s1 []Instruction, s2 []Instruction) int { + n := min(len(s1), len(s2)) + for i := 0; i < n; i++ { + if CompareInstructions(s1[i], s2[i]) != 0 { + return i + } + } + return n +}