upgrade .SerializeInstructionSequences() to the crazy checkpointing scheme.

helpers.go

@@ -0,0 +1,52 @@
+package opentimestamps
+
+import (
+	"strings"
+
+	"golang.org/x/exp/slices"
+)
+
+// CompareInstructions returns negative if a<b, 0 if a=b and positive if a>b.
+// It considers an operation smaller than an attestation, a pending attestation smaller than a Bitcoin attestation.
+// It orders operations by their tag byte and then by their argument.
+func CompareInstructions(a, b Instruction) int {
+	if a.Operation != nil {
+		if b.Attestation != nil {
+			// a is an operation but b is an attestation, b is bigger
+			return -1
+		}
+		if a.Operation == b.Operation {
+			// if both are the same operation sort by the argument
+			return slices.Compare(a.Argument, b.Argument)
+		}
+
+		// sort by the operation
+		if a.Operation.Tag < b.Operation.Tag {
+			return -1
+		} else if a.Operation.Tag > b.Operation.Tag {
+			return 1
+		} else {
+			return 0
+		}
+	} else if a.Attestation != nil && b.Attestation == nil {
+		// a is an attestation but b is not, a is bigger
+		return 1
+	} else if a.Attestation != nil && b.Attestation != nil {
+		// both are attestations
+		if a.Attestation.BitcoinBlockHeight == 0 && b.Attestation.BitcoinBlockHeight == 0 {
+			// none are bitcoin attestations
+			return strings.Compare(a.Attestation.CalendarServerURL, b.Attestation.CalendarServerURL)
+		}
+		if a.Attestation.BitcoinBlockHeight != 0 && b.Attestation.BitcoinBlockHeight != 0 {
+			// both are bitcoin attestations
+			return int(b.Attestation.BitcoinBlockHeight - a.Attestation.BitcoinBlockHeight)
+		}
+
+		// one is bitcoin and the other is not -- compare by bitcoin block,
+		// but reverse the result since the one with 0 should not be considered bigger
+		return -1 * int(b.Attestation.BitcoinBlockHeight-a.Attestation.BitcoinBlockHeight)
+	} else {
+		// this shouldn't happen
+		return 0
+	}
+}

ots.go

@@ -52,10 +52,10 @@ var tags = map[byte]*Operation{
 	0x67: {"keccak256", 0x67, false, func(curr []byte, arg []byte) []byte { panic("keccak256 not implemented") }},
 }
 
-// A Timestamp is basically the content of an .ots file: it has an initial digest and
+// A File represents the parsed content of an .ots file: it has an initial digest and
 // a series of sequences of instructions. Each sequence must be evaluated separately, applying the operations
 // on top of each other, starting with the .Digest until they end on an attestation.
-type Timestamp struct {
+type File struct {
 	Digest       []byte
 	Instructions []Sequence
 }
@@ -69,35 +69,6 @@ type Instruction struct {
 	*Attestation
 }
 
-func (a Instruction) Equal(b Instruction) bool {
-	if a.Operation != nil {
-		if a.Operation == b.Operation && slices.Equal(a.Argument, b.Argument) {
-			return true
-		} else {
-			return false
-		}
-	} else if a.Attestation != nil {
-		if b.Attestation == nil {
-			return false
-		}
-		if a.Attestation.BitcoinBlockHeight != 0 &&
-			a.Attestation.BitcoinBlockHeight == b.Attestation.BitcoinBlockHeight {
-			return true
-		}
-		if a.Attestation.CalendarServerURL != "" &&
-			a.Attestation.CalendarServerURL == b.Attestation.CalendarServerURL {
-			return true
-		}
-		return false
-	} else {
-		// a is nil -- this is already broken but whatever
-		if b.Attestation == nil && b.Operation == nil {
-			return true
-		}
-		return false
-	}
-}
-
 type Sequence []Instruction
 
 func (seq Sequence) Compute(initial []byte) []byte {
@@ -111,7 +82,7 @@ func (seq Sequence) Compute(initial []byte) []byte {
 	return current
 }
 
-func (ts Timestamp) GetPendingSequences() []Sequence {
+func (ts File) GetPendingSequences() []Sequence {
 	bitcoin := ts.GetBitcoinAttestedSequences()
 
 	results := make([]Sequence, 0, len(ts.Instructions))
@@ -125,7 +96,7 @@ func (ts Timestamp) GetPendingSequences() []Sequence {
 					continue
 				}
 
-				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return a.Equal(b) }) {
+				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return CompareInstructions(a, b) == 0 }) {
 					goto thisSequenceIsAlreadyConfirmed
 				}
 			}
@@ -141,7 +112,7 @@ func (ts Timestamp) GetPendingSequences() []Sequence {
 	return results
 }
 
-func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence {
+func (ts File) GetBitcoinAttestedSequences() []Sequence {
 	results := make([]Sequence, 0, len(ts.Instructions))
 	for _, seq := range ts.Instructions {
 		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.BitcoinBlockHeight > 0 {
@@ -151,7 +122,7 @@ func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence {
 	return results
 }
 
-func (ts Timestamp) Human() string {
+func (ts File) Human() string {
 	strs := make([]string, 0, 100)
 	strs = append(strs, fmt.Sprintf("file digest: %x", ts.Digest))
 	strs = append(strs, fmt.Sprintf("hashed with: sha256"))
@@ -176,7 +147,7 @@ func (ts Timestamp) Human() string {
 	return strings.Join(strs, "\n")
 }
 
-func (ts Timestamp) SerializeToFile() []byte {
+func (ts File) SerializeToFile() []byte {
 	data := make([]byte, 0, 5050)
 	data = append(data, headerMagic...)
 	data = appendVarUint(data, 1)
@@ -186,43 +157,77 @@ func (ts Timestamp) SerializeToFile() []byte {
 	return data
 }
 
-func (ts Timestamp) SerializeInstructionSequences() []byte {
+func (ts File) SerializeInstructionSequences() []byte {
-	data := make([]byte, 0, 5000)
+	sequences := make([]Sequence, len(ts.Instructions))
-	for i, seq := range ts.Instructions {
+	copy(sequences, ts.Instructions)
-		for _, inst := range seq {
+
+	// first we sort everything so the checkpoint stuff makes sense
+	slices.SortFunc(sequences, func(a, b Sequence) int { return slices.CompareFunc(a, b, CompareInstructions) })
+
+	// checkpoints we may leave to the next people
+	sequenceCheckpoints := make([][]int, len(sequences))
+	for s1 := range sequences {
+		// keep an ordered slice of all the checkpoints we will potentially leave during our write journey for this sequence
+		checkpoints := make([]int, 0, len(sequences[s1]))
+		for s2 := s1 + 1; s2 < len(sequences); s2++ {
+			chp := getCommonPrefixIndex(sequences[s1], sequences[s2])
+			if pos, found := slices.BinarySearch(checkpoints, chp); !found {
+				checkpoints = append(checkpoints, -1)        // make room
+				copy(checkpoints[pos+1:], checkpoints[pos:]) // move elements to the right
+				checkpoints[pos] = chp                       // insert this
+			}
+		}
+		sequenceCheckpoints[s1] = checkpoints
+	}
+
+	// now actually go through the sequences writing them
+	result := make([]byte, 0, 500)
+	for s, seq := range sequences {
+		startingAt := 0
+		if s > 0 {
+			// we will always start at the last checkpoint left by the previous sequence
+			startingAt = sequenceCheckpoints[s-1][len(sequenceCheckpoints[s-1])-1]
+		}
+
+		for i := startingAt; i < len(seq); i++ {
+			// before writing anything, decide if we wanna leave a checkpoint here
+			for _, chk := range sequenceCheckpoints[s] {
+				if chk == i {
+					// leave a checkpoint
+					result = append(result, 0xff)
+				}
+			}
+
+			inst := seq[i]
 			if inst.Operation != nil {
 				// write normal operation
-				data = append(data, inst.Operation.Tag)
+				result = append(result, inst.Operation.Tag)
 				if inst.Operation.Binary {
-					data = appendVarBytes(data, inst.Argument)
+					result = appendVarBytes(result, inst.Argument)
 				}
 			} else if inst.Attestation != nil {
 				// write attestation record
-				data = append(data, 0x00)
+				result = append(result, 0x00)
 				{
-					// will use a new buffer for the actual attestation data
+					// will use a new buffer for the actual attestation result
 					abuf := make([]byte, 0, 100)
 					if inst.BitcoinBlockHeight != 0 {
-						data = append(data, bitcoinMagic...) // this goes in the main data buffer
+						result = append(result, bitcoinMagic...) // this goes in the main result buffer
 						abuf = appendVarUint(abuf, inst.BitcoinBlockHeight)
 					} else if inst.CalendarServerURL != "" {
-						data = append(data, pendingMagic...) // this goes in the main data buffer
+						result = append(result, pendingMagic...) // this goes in the main result buffer
 						abuf = appendVarBytes(abuf, []byte(inst.CalendarServerURL))
 					} else {
 						panic(fmt.Sprintf("invalid attestation: %v", inst))
 					}
-					data = appendVarBytes(data, abuf) // we append that data as varbytes
+					result = appendVarBytes(result, abuf) // we append that result as varbytes
 				}
 			} else {
 				panic(fmt.Sprintf("invalid instruction: %v", inst))
 			}
 		}
-		if i+1 < len(ts.Instructions) {
-			// write separator and start a new sequence of instructions
-			data = append(data, 0xff)
-		}
 	}
-	return data
+	return result
 }
 
 type Attestation struct {

parsers.go

@@ -19,7 +19,7 @@ func parseCalendarServerResponse(buf Buffer) (Sequence, error) {
 	return seqs[0], nil
 }
 
-func parseOTSFile(buf Buffer) (*Timestamp, error) {
+func parseOTSFile(buf Buffer) (*File, error) {
 	// read magic
 	// read version [1 byte]
 	// read crypto operation for file digest [1 byte]
@@ -47,7 +47,7 @@ func parseOTSFile(buf Buffer) (*Timestamp, error) {
 		return nil, fmt.Errorf("failed to read 32-byte digest: %w", err)
 	}
 
-	ts := &Timestamp{
+	ts := &File{
 		Digest: digest,
 	}
 
@@ -102,7 +102,7 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) {
 			if err != nil {
 				return nil, fmt.Errorf("failed to read attestation bytes: %w", err)
 			}
-			abuf := NewBuffer(this)
+			abuf := newBuffer(this)
 
 			switch {
 			case slices.Equal(magic, pendingMagic):
@@ -131,17 +131,17 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) {
 			ncheckpoints := len(checkpoints)
 			if ncheckpoints > 0 {
 				// use this checkpoint as the starting point for the next block
-				cp := checkpoints[ncheckpoints-1]
+				chp := checkpoints[ncheckpoints-1]
 				checkpoints = checkpoints[0 : ncheckpoints-1] // remove this from the stack
-				seqs = append(seqs, cp)
+				seqs = append(seqs, chp)
 				currInstructionsBlock++
 			}
 		} else if tag == 0xff {
 			// pick up a checkpoint to be used later
 			currentBlock := seqs[currInstructionsBlock]
-			cp := make([]Instruction, len(currentBlock))
+			chp := make([]Instruction, len(currentBlock))
-			copy(cp, currentBlock)
+			copy(chp, currentBlock)
-			checkpoints = append(checkpoints, cp)
+			checkpoints = append(checkpoints, chp)
 		} else {
 			// a new operation in this block
 			inst, err := readInstruction(buf, tag)

stamp.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 )
 
-func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp, error) {
+func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (Sequence, error) {
 	body := bytes.NewBuffer(digest[:])
 	req, err := http.NewRequestWithContext(ctx, "POST", normalizeUrl(calendarUrl)+"/digest", body)
 	if err != nil {
@@ -29,19 +29,16 @@ func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp
 	}
 	resp.Body.Close()
 
-	seq, err := parseCalendarServerResponse(NewBuffer(full))
+	seq, err := parseCalendarServerResponse(newBuffer(full))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response from '%s': %w", calendarUrl, err)
 	}
 
-	return &Timestamp{
+	return seq, nil
-		Digest:       digest[:],
-		Instructions: []Sequence{seq},
-	}, nil
 }
 
-func ReadFromFile(data []byte) (*Timestamp, error) {
+func ReadFromFile(data []byte) (*File, error) {
-	return parseOTSFile(NewBuffer(data))
+	return parseOTSFile(newBuffer(data))
 }
 
 func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, error) {
@@ -72,7 +69,7 @@ func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, erro
 	}
 	resp.Body.Close()
 
-	newSeq, err := parseCalendarServerResponse(NewBuffer(body))
+	newSeq, err := parseCalendarServerResponse(newBuffer(body))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response from '%s': %w", attestation.CalendarServerURL, err)
 	}

utils.go

@@ -20,7 +20,7 @@ type Buffer struct {
 	buf []byte
 }
 
-func NewBuffer(buf []byte) Buffer {
+func newBuffer(buf []byte) Buffer {
 	zero := 0
 	return Buffer{&zero, buf}
 }
@@ -99,3 +99,13 @@ func appendVarBytes(buf []byte, value []byte) []byte {
 	buf = append(buf, value...)
 	return buf
 }
+
+func getCommonPrefixIndex(s1 []Instruction, s2 []Instruction) int {
+	n := min(len(s1), len(s2))
+	for i := 0; i < n; i++ {
+		if CompareInstructions(s1[i], s2[i]) != 0 {
+			return i
+		}
+	}
+	return n
+}