a readme.

README.md

@@ -2,6 +2,80 @@
 
 Interact with calendar servers, create and verify OTS attestations.
 
+# How to use
+
+Full documentation at https://pkg.go.dev/github.com/nbd-wtf/opentimestamps. See some commented pseudocode below (you probably should not try to run it as it is).
+
+```go
+package main
+
+import "github.com/nbd-wtf/opentimestamps"
+
+func main () {
+    // create a timestamp at a specific calendar server
+    hash := sha256.Sum256([]byte{1,2,3,4,5,6})
+    seq, _ := opentimestamps.Stamp(context.Background(), "https://alice.btc.calendar.opentimestamps.org/", hash)
+
+    // you can just call .Upgrade() to get the upgraded sequence (or an error if not yet available)
+    upgradedSeq, err := seq.Upgrade(context.Background(), hash[:])
+    if err != nil {
+        fmt.Println("wait more")
+    }
+
+    // a File is a struct that represents the content of an .ots file, which contains the initial digest and any number of sequences
+    file := File{
+        Digest: hash,
+        Sequences: []Sequence{seq},
+    }
+
+    // it can be written to disk
+    os.WriteFile("file.ots", file.SerializeToFile(), 0644)
+
+    // or printed in human-readable format
+    fmt.Println(file.Human())
+
+    // sequences are always composed of a bunch of operation instructions -- these can be, for example, "append", "prepend", "sha256"
+    fmt.Println(seq[0].Operation.Name) // "append"
+    fmt.Println(seq[1].Operation.Name) // "sha256"
+    fmt.Println(seq[2].Operation.Name) // "prepend"
+
+    // "prepend" and "append" are "binary", i.e. they take an argument
+    fmt.Println(hex.EncodeToString(seq[2].Argument)) // "c40fe258f9b828a0b5a7"
+
+    // all these instructions can be executed in order, starting from the initial hash
+    result := seq.Compute(hash) // this is the value we send to the calendar server in order to get the upgraded sequence on .Upgrade()
+    finalResult := upgradedSeq.Compute(hash) // this should be the merkle root of a bitcoin block if this sequence is upgraded
+
+    // each sequence always ends in an "attestation"
+    // it can be either a pending attestation, i.e. a reference to a calendar server from which we will upgrade this sequence later
+    fmt.Println(seq[len(seq)-1].Attestation.CalendarServerURL) // "https://alice.btc.calendar.opentimestamps.org/"
+    // or it can be a reference to a bitcoin block, the merkle root of which we will check against the result of Compute() for verifying
+    fmt.Println(upgradedSeq[len(upgradedSeq)-1].Attestation.BitcoinBlockHeight) // 810041
+
+    // speaking of verifying, this is how we do it:
+    // first we need some source of bitcoin blocks,
+    var bitcoin opentimestamps.Bitcoin
+    if useLocallyRunningBitcoindNode {
+        // it can be either a locally running bitcoind node
+        bitcoin, _ = opentimestamps.NewBitcoindInterface(rpcclient.ConnConfig{
+            User:         "nakamoto",
+            Pass:         "mumbojumbo",
+            HTTPPostMode: true,
+        })
+    } else {
+        // or an esplora HTTP endpoint
+        bitcoin = opentimestamps.NewEsploraClient("https://blockstream.info/api")
+    }
+
+    // then we pass that to a sequence
+    if err := upgradedSeq.Verify(bitcoin, hash); err == nil {
+        fmt.Println("it works!")
+    }
+}
+```
+
+You can also take a look at [`ots`](https://github.com/fiatjaf/ots), a simple CLI to OpenTimestamps which is basically a wrapper over this library.
+
 # License
 
 Public Domain

go.mod

@@ -1,10 +1,11 @@
-module github.com/fiatjaf/opentimestamps
+module github.com/nbd-wtf/opentimestamps
 
 go 1.21
 
 require (
 	github.com/btcsuite/btcd v0.23.4
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1
+	golang.org/x/crypto v0.13.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 )
 
@@ -17,6 +18,5 @@ require (
 	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 )

helpers.go

@@ -0,0 +1,52 @@
+package opentimestamps
+
+import (
+	"strings"
+
+	"golang.org/x/exp/slices"
+)
+
+// CompareInstructions returns negative if a<b, 0 if a=b and positive if a>b.
+// It considers an operation smaller than an attestation, a pending attestation smaller than a Bitcoin attestation.
+// It orders operations by their tag byte and then by their argument.
+func CompareInstructions(a, b Instruction) int {
+	if a.Operation != nil {
+		if b.Attestation != nil {
+			// a is an operation but b is an attestation, a is bigger
+			return +1
+		}
+		if a.Operation == b.Operation {
+			// if both are the same operation sort by the argument
+			return slices.Compare(a.Argument, b.Argument)
+		}
+
+		// sort by the operation
+		if a.Operation.Tag < b.Operation.Tag {
+			return -1
+		} else if a.Operation.Tag > b.Operation.Tag {
+			return 1
+		} else {
+			return 0
+		}
+	} else if a.Attestation != nil && b.Attestation == nil {
+		// a is an attestation but b is not, b is bigger
+		return -1
+	} else if a.Attestation != nil && b.Attestation != nil {
+		// both are attestations
+		if a.Attestation.BitcoinBlockHeight == 0 && b.Attestation.BitcoinBlockHeight == 0 {
+			// none are bitcoin attestations
+			return strings.Compare(a.Attestation.CalendarServerURL, b.Attestation.CalendarServerURL)
+		}
+		if a.Attestation.BitcoinBlockHeight != 0 && b.Attestation.BitcoinBlockHeight != 0 {
+			// both are bitcoin attestations
+			return int(b.Attestation.BitcoinBlockHeight - a.Attestation.BitcoinBlockHeight)
+		}
+
+		// one is bitcoin and the other is not -- compare by bitcoin block,
+		// but reverse the result since the one with 0 should not be considered bigger
+		return -1 * int(b.Attestation.BitcoinBlockHeight-a.Attestation.BitcoinBlockHeight)
+	} else {
+		// this shouldn't happen
+		return 0
+	}
+}

ots.go

@@ -52,12 +52,12 @@ var tags = map[byte]*Operation{
 	0x67: {"keccak256", 0x67, false, func(curr []byte, arg []byte) []byte { panic("keccak256 not implemented") }},
 }
 
-// A Timestamp is basically the content of an .ots file: it has an initial digest and
+// A File represents the parsed content of an .ots file: it has an initial digest and
 // a series of sequences of instructions. Each sequence must be evaluated separately, applying the operations
 // on top of each other, starting with the .Digest until they end on an attestation.
-type Timestamp struct {
-	Digest       []byte
-	Instructions []Sequence
+type File struct {
+	Digest    []byte
+	Sequences []Sequence
 }
 
 // a Instruction can be an operation like "append" or "prepend" (this will be the case when .Operation != nil)
@@ -69,35 +69,6 @@ type Instruction struct {
 	*Attestation
 }
 
-func (a Instruction) Equal(b Instruction) bool {
-	if a.Operation != nil {
-		if a.Operation == b.Operation && slices.Equal(a.Argument, b.Argument) {
-			return true
-		} else {
-			return false
-		}
-	} else if a.Attestation != nil {
-		if b.Attestation == nil {
-			return false
-		}
-		if a.Attestation.BitcoinBlockHeight != 0 &&
-			a.Attestation.BitcoinBlockHeight == b.Attestation.BitcoinBlockHeight {
-			return true
-		}
-		if a.Attestation.CalendarServerURL != "" &&
-			a.Attestation.CalendarServerURL == b.Attestation.CalendarServerURL {
-			return true
-		}
-		return false
-	} else {
-		// a is nil -- this is already broken but whatever
-		if b.Attestation == nil && b.Operation == nil {
-			return true
-		}
-		return false
-	}
-}
-
 type Sequence []Instruction
 
 func (seq Sequence) Compute(initial []byte) []byte {
@@ -111,11 +82,11 @@ func (seq Sequence) Compute(initial []byte) []byte {
 	return current
 }
 
-func (ts Timestamp) GetPendingSequences() []Sequence {
+func (ts File) GetPendingSequences() []Sequence {
 	bitcoin := ts.GetBitcoinAttestedSequences()
 
-	results := make([]Sequence, 0, len(ts.Instructions))
-	for _, seq := range ts.Instructions {
+	results := make([]Sequence, 0, len(ts.Sequences))
+	for _, seq := range ts.Sequences {
 		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.CalendarServerURL != "" {
 			// this is a calendar sequence, fine
 			// now we check if this same sequence isn't contained in a bigger one that contains a bitcoin attestation
@@ -125,7 +96,7 @@ func (ts Timestamp) GetPendingSequences() []Sequence {
 					continue
 				}
 
-				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return a.Equal(b) }) {
+				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return CompareInstructions(a, b) == 0 }) {
 					goto thisSequenceIsAlreadyConfirmed
 				}
 			}
@@ -141,9 +112,9 @@ func (ts Timestamp) GetPendingSequences() []Sequence {
 	return results
 }
 
-func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence {
-	results := make([]Sequence, 0, len(ts.Instructions))
-	for _, seq := range ts.Instructions {
+func (ts File) GetBitcoinAttestedSequences() []Sequence {
+	results := make([]Sequence, 0, len(ts.Sequences))
+	for _, seq := range ts.Sequences {
 		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.BitcoinBlockHeight > 0 {
 			results = append(results, seq)
 		}
@@ -151,12 +122,12 @@ func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence {
 	return results
 }
 
-func (ts Timestamp) Human() string {
+func (ts File) Human() string {
 	strs := make([]string, 0, 100)
 	strs = append(strs, fmt.Sprintf("file digest: %x", ts.Digest))
 	strs = append(strs, fmt.Sprintf("hashed with: sha256"))
 	strs = append(strs, "instruction sequences:")
-	for _, seq := range ts.Instructions {
+	for _, seq := range ts.Sequences {
 		strs = append(strs, "~>")
 		for _, inst := range seq {
 			line := "  "
@@ -176,7 +147,7 @@ func (ts Timestamp) Human() string {
 	return strings.Join(strs, "\n")
 }
 
-func (ts Timestamp) SerializeToFile() []byte {
+func (ts File) SerializeToFile() []byte {
 	data := make([]byte, 0, 5050)
 	data = append(data, headerMagic...)
 	data = appendVarUint(data, 1)
@@ -186,43 +157,77 @@ func (ts Timestamp) SerializeToFile() []byte {
 	return data
 }
 
-func (ts Timestamp) SerializeInstructionSequences() []byte {
-	data := make([]byte, 0, 5000)
-	for i, seq := range ts.Instructions {
-		for _, inst := range seq {
+func (ts File) SerializeInstructionSequences() []byte {
+	sequences := make([]Sequence, len(ts.Sequences))
+	copy(sequences, ts.Sequences)
+
+	// first we sort everything so the checkpoint stuff makes sense
+	slices.SortFunc(sequences, func(a, b Sequence) int { return slices.CompareFunc(a, b, CompareInstructions) })
+
+	// checkpoints we may leave to the next people
+	sequenceCheckpoints := make([][]int, len(sequences))
+	for s1 := range sequences {
+		// keep an ordered slice of all the checkpoints we will potentially leave during our write journey for this sequence
+		checkpoints := make([]int, 0, len(sequences[s1]))
+		for s2 := s1 + 1; s2 < len(sequences); s2++ {
+			chp := getCommonPrefixIndex(sequences[s1], sequences[s2])
+			if pos, found := slices.BinarySearch(checkpoints, chp); !found {
+				checkpoints = append(checkpoints, -1)        // make room
+				copy(checkpoints[pos+1:], checkpoints[pos:]) // move elements to the right
+				checkpoints[pos] = chp                       // insert this
+			}
+		}
+		sequenceCheckpoints[s1] = checkpoints
+	}
+
+	// now actually go through the sequences writing them
+	result := make([]byte, 0, 500)
+	for s, seq := range sequences {
+		startingAt := 0
+		if s > 0 {
+			// we will always start at the last checkpoint left by the previous sequence
+			startingAt = sequenceCheckpoints[s-1][len(sequenceCheckpoints[s-1])-1]
+		}
+
+		for i := startingAt; i < len(seq); i++ {
+			// before writing anything, decide if we wanna leave a checkpoint here
+			for _, chk := range sequenceCheckpoints[s] {
+				if chk == i {
+					// leave a checkpoint
+					result = append(result, 0xff)
+				}
+			}
+
+			inst := seq[i]
 			if inst.Operation != nil {
 				// write normal operation
-				data = append(data, inst.Operation.Tag)
+				result = append(result, inst.Operation.Tag)
 				if inst.Operation.Binary {
-					data = appendVarBytes(data, inst.Argument)
+					result = appendVarBytes(result, inst.Argument)
 				}
 			} else if inst.Attestation != nil {
 				// write attestation record
-				data = append(data, 0x00)
+				result = append(result, 0x00)
 				{
-					// will use a new buffer for the actual attestation data
+					// will use a new buffer for the actual attestation result
 					abuf := make([]byte, 0, 100)
 					if inst.BitcoinBlockHeight != 0 {
-						data = append(data, bitcoinMagic...) // this goes in the main data buffer
+						result = append(result, bitcoinMagic...) // this goes in the main result buffer
 						abuf = appendVarUint(abuf, inst.BitcoinBlockHeight)
 					} else if inst.CalendarServerURL != "" {
-						data = append(data, pendingMagic...) // this goes in the main data buffer
+						result = append(result, pendingMagic...) // this goes in the main result buffer
 						abuf = appendVarBytes(abuf, []byte(inst.CalendarServerURL))
 					} else {
 						panic(fmt.Sprintf("invalid attestation: %v", inst))
 					}
-					data = appendVarBytes(data, abuf) // we append that data as varbytes
+					result = appendVarBytes(result, abuf) // we append that result as varbytes
 				}
 			} else {
 				panic(fmt.Sprintf("invalid instruction: %v", inst))
 			}
 		}
-		if i+1 < len(ts.Instructions) {
-			// write separator and start a new sequence of instructions
-			data = append(data, 0xff)
-		}
 	}
-	return data
+	return result
 }
 
 type Attestation struct {

parsers.go

@@ -19,7 +19,7 @@ func parseCalendarServerResponse(buf Buffer) (Sequence, error) {
 	return seqs[0], nil
 }
 
-func parseOTSFile(buf Buffer) (*Timestamp, error) {
+func parseOTSFile(buf Buffer) (*File, error) {
 	// read magic
 	// read version [1 byte]
 	// read crypto operation for file digest [1 byte]
@@ -47,14 +47,14 @@ func parseOTSFile(buf Buffer) (*Timestamp, error) {
 		return nil, fmt.Errorf("failed to read 32-byte digest: %w", err)
 	}
 
-	ts := &Timestamp{
+	ts := &File{
 		Digest: digest,
 	}
 
 	if seqs, err := parseTimestamp(buf); err != nil {
 		return nil, err
 	} else {
-		ts.Instructions = seqs
+		ts.Sequences = seqs
 	}
 
 	return ts, nil
@@ -102,7 +102,7 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) {
 			if err != nil {
 				return nil, fmt.Errorf("failed to read attestation bytes: %w", err)
 			}
-			abuf := NewBuffer(this)
+			abuf := newBuffer(this)
 
 			switch {
 			case slices.Equal(magic, pendingMagic):
@@ -131,17 +131,17 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) {
 			ncheckpoints := len(checkpoints)
 			if ncheckpoints > 0 {
 				// use this checkpoint as the starting point for the next block
-				cp := checkpoints[ncheckpoints-1]
+				chp := checkpoints[ncheckpoints-1]
 				checkpoints = checkpoints[0 : ncheckpoints-1] // remove this from the stack
-				seqs = append(seqs, cp)
+				seqs = append(seqs, chp)
 				currInstructionsBlock++
 			}
 		} else if tag == 0xff {
 			// pick up a checkpoint to be used later
 			currentBlock := seqs[currInstructionsBlock]
-			cp := make([]Instruction, len(currentBlock))
-			copy(cp, currentBlock)
-			checkpoints = append(checkpoints, cp)
+			chp := make([]Instruction, len(currentBlock))
+			copy(chp, currentBlock)
+			checkpoints = append(checkpoints, chp)
 		} else {
 			// a new operation in this block
 			inst, err := readInstruction(buf, tag)

stamp.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 )
 
-func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp, error) {
+func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (Sequence, error) {
 	body := bytes.NewBuffer(digest[:])
 	req, err := http.NewRequestWithContext(ctx, "POST", normalizeUrl(calendarUrl)+"/digest", body)
 	if err != nil {
@@ -29,19 +29,16 @@ func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp
 	}
 	resp.Body.Close()
 
-	seq, err := parseCalendarServerResponse(NewBuffer(full))
+	seq, err := parseCalendarServerResponse(newBuffer(full))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response from '%s': %w", calendarUrl, err)
 	}
 
-	return &Timestamp{
-		Digest:       digest[:],
-		Instructions: []Sequence{seq},
-	}, nil
+	return seq, nil
 }
 
-func ReadFromFile(data []byte) (*Timestamp, error) {
-	return parseOTSFile(NewBuffer(data))
+func ReadFromFile(data []byte) (*File, error) {
+	return parseOTSFile(newBuffer(data))
 }
 
 func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, error) {
@@ -72,7 +69,7 @@ func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, erro
 	}
 	resp.Body.Close()
 
-	newSeq, err := parseCalendarServerResponse(NewBuffer(body))
+	newSeq, err := parseCalendarServerResponse(newBuffer(body))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response from '%s': %w", attestation.CalendarServerURL, err)
 	}

utils.go

@@ -20,7 +20,7 @@ type Buffer struct {
 	buf []byte
 }
 
-func NewBuffer(buf []byte) Buffer {
+func newBuffer(buf []byte) Buffer {
 	zero := 0
 	return Buffer{&zero, buf}
 }
@@ -99,3 +99,13 @@ func appendVarBytes(buf []byte, value []byte) []byte {
 	buf = append(buf, value...)
 	return buf
 }
+
+func getCommonPrefixIndex(s1 []Instruction, s2 []Instruction) int {
+	n := min(len(s1), len(s2))
+	for i := 0; i < n; i++ {
+		if CompareInstructions(s1[i], s2[i]) != 0 {
+			return i
+		}
+	}
+	return n
+}

verifier.go

@@ -36,7 +36,7 @@ func (seq Sequence) Verify(bitcoin Bitcoin, initial []byte) error {
 	merkleRoot := blockHeader.MerkleRoot[:]
 
 	result := seq.Compute(initial)
-	if slices.Equal(result, merkleRoot) {
+	if !slices.Equal(result, merkleRoot) {
 		return fmt.Errorf("sequence result '%x' doesn't match the bitcoin merkle root for block %d: %x",
 			result, att.BitcoinBlockHeight, merkleRoot)
 	}