add license so pkg.go.dev works.

LICENSE

@@ -0,0 +1,7 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
+
+In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -2,6 +2,80 @@
 
 Interact with calendar servers, create and verify OTS attestations.
 
+# How to use
+
+Full documentation at https://pkg.go.dev/github.com/nbd-wtf/opentimestamps. See some commented pseudocode below (you probably should not try to run it as it is).
+
+```go
+package main
+
+import "github.com/nbd-wtf/opentimestamps"
+
+func main () {
+    // create a timestamp at a specific calendar server
+    hash := sha256.Sum256([]byte{1,2,3,4,5,6})
+    seq, _ := opentimestamps.Stamp(context.Background(), "https://alice.btc.calendar.opentimestamps.org/", hash)
+
+    // you can just call .Upgrade() to get the upgraded sequence (or an error if not yet available)
+    upgradedSeq, err := seq.Upgrade(context.Background(), hash[:])
+    if err != nil {
+        fmt.Println("wait more")
+    }
+
+    // a File is a struct that represents the content of an .ots file, which contains the initial digest and any number of sequences
+    file := File{
+        Digest: hash,
+        Sequences: []Sequence{seq},
+    }
+
+    // it can be written to disk
+    os.WriteFile("file.ots", file.SerializeToFile(), 0644)
+
+    // or printed in human-readable format
+    fmt.Println(file.Human())
+
+    // sequences are always composed of a bunch of operation instructions -- these can be, for example, "append", "prepend", "sha256"
+    fmt.Println(seq[0].Operation.Name) // "append"
+    fmt.Println(seq[1].Operation.Name) // "sha256"
+    fmt.Println(seq[2].Operation.Name) // "prepend"
+
+    // "prepend" and "append" are "binary", i.e. they take an argument
+    fmt.Println(hex.EncodeToString(seq[2].Argument)) // "c40fe258f9b828a0b5a7"
+
+    // all these instructions can be executed in order, starting from the initial hash
+    result := seq.Compute(hash) // this is the value we send to the calendar server in order to get the upgraded sequence on .Upgrade()
+    finalResult := upgradedSeq.Compute(hash) // this should be the merkle root of a bitcoin block if this sequence is upgraded
+
+    // each sequence always ends in an "attestation"
+    // it can be either a pending attestation, i.e. a reference to a calendar server from which we will upgrade this sequence later
+    fmt.Println(seq[len(seq)-1].Attestation.CalendarServerURL) // "https://alice.btc.calendar.opentimestamps.org/"
+    // or it can be a reference to a bitcoin block, the merkle root of which we will check against the result of Compute() for verifying
+    fmt.Println(upgradedSeq[len(upgradedSeq)-1].Attestation.BitcoinBlockHeight) // 810041
+
+    // speaking of verifying, this is how we do it:
+    // first we need some source of bitcoin blocks,
+    var bitcoin opentimestamps.Bitcoin
+    if useLocallyRunningBitcoindNode {
+        // it can be either a locally running bitcoind node
+        bitcoin, _ = opentimestamps.NewBitcoindInterface(rpcclient.ConnConfig{
+            User:         "nakamoto",
+            Pass:         "mumbojumbo",
+            HTTPPostMode: true,
+        })
+    } else {
+        // or an esplora HTTP endpoint
+        bitcoin = opentimestamps.NewEsploraClient("https://blockstream.info/api")
+    }
+
+    // then we pass that to a sequence
+    if err := upgradedSeq.Verify(bitcoin, hash); err == nil {
+        fmt.Println("it works!")
+    }
+}
+```
+
+You can also take a look at [`ots`](https://github.com/fiatjaf/ots), a simple CLI to OpenTimestamps which is basically a wrapper over this library.
+
 # License
 
 Public Domain

go.mod

@@ -1,10 +1,11 @@
-module github.com/fiatjaf/opentimestamps
+module github.com/nbd-wtf/opentimestamps
 
 go 1.21
 
 require (
 	github.com/btcsuite/btcd v0.23.4
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1
+	golang.org/x/crypto v0.13.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 )
 
@@ -17,6 +18,5 @@ require (
 	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 )

helpers.go

@@ -0,0 +1,52 @@
+package opentimestamps
+
+import (
+	"strings"
+
+	"golang.org/x/exp/slices"
+)
+
+// CompareInstructions returns negative if a<b, 0 if a=b and positive if a>b.
+// It considers an operation smaller than an attestation, a pending attestation smaller than a Bitcoin attestation.
+// It orders operations by their tag byte and then by their argument.
+func CompareInstructions(a, b Instruction) int {
+	if a.Operation != nil {
+		if b.Attestation != nil {
+			// a is an operation but b is an attestation, a is bigger
+			return +1
+		}
+		if a.Operation == b.Operation {
+			// if both are the same operation sort by the argument
+			return slices.Compare(a.Argument, b.Argument)
+		}
+
+		// sort by the operation
+		if a.Operation.Tag < b.Operation.Tag {
+			return -1
+		} else if a.Operation.Tag > b.Operation.Tag {
+			return 1
+		} else {
+			return 0
+		}
+	} else if a.Attestation != nil && b.Attestation == nil {
+		// a is an attestation but b is not, b is bigger
+		return -1
+	} else if a.Attestation != nil && b.Attestation != nil {
+		// both are attestations
+		if a.Attestation.BitcoinBlockHeight == 0 && b.Attestation.BitcoinBlockHeight == 0 {
+			// none are bitcoin attestations
+			return strings.Compare(a.Attestation.CalendarServerURL, b.Attestation.CalendarServerURL)
+		}
+		if a.Attestation.BitcoinBlockHeight != 0 && b.Attestation.BitcoinBlockHeight != 0 {
+			// both are bitcoin attestations
+			return int(b.Attestation.BitcoinBlockHeight - a.Attestation.BitcoinBlockHeight)
+		}
+
+		// one is bitcoin and the other is not -- compare by bitcoin block,
+		// but reverse the result since the one with 0 should not be considered bigger
+		return -1 * int(b.Attestation.BitcoinBlockHeight-a.Attestation.BitcoinBlockHeight)
+	} else {
+		// this shouldn't happen
+		return 0
+	}
+}

ots.go

@@ -52,12 +52,12 @@ var tags = map[byte]*Operation{
 	0x67: {"keccak256", 0x67, false, func(curr []byte, arg []byte) []byte { panic("keccak256 not implemented") }},
 }
 
-// A Timestamp is basically the content of an .ots file: it has an initial digest and
+// A File represents the parsed content of an .ots file: it has an initial digest and
 // a series of sequences of instructions. Each sequence must be evaluated separately, applying the operations
 // on top of each other, starting with the .Digest until they end on an attestation.
-type Timestamp struct {
-	Digest       []byte
-	Instructions []Sequence
+type File struct {
+	Digest    []byte
+	Sequences []Sequence
 }
 
 // a Instruction can be an operation like "append" or "prepend" (this will be the case when .Operation != nil)
@@ -69,35 +69,6 @@ type Instruction struct {
 	*Attestation
 }
 
-func (a Instruction) Equal(b Instruction) bool {
-	if a.Operation != nil {
-		if a.Operation == b.Operation && slices.Equal(a.Argument, b.Argument) {
-			return true
-		} else {
-			return false
-		}
-	} else if a.Attestation != nil {
-		if b.Attestation == nil {
-			return false
-		}
-		if a.Attestation.BitcoinBlockHeight != 0 &&
-			a.Attestation.BitcoinBlockHeight == b.Attestation.BitcoinBlockHeight {
-			return true
-		}
-		if a.Attestation.CalendarServerURL != "" &&
-			a.Attestation.CalendarServerURL == b.Attestation.CalendarServerURL {
-			return true
-		}
-		return false
-	} else {
-		// a is nil -- this is already broken but whatever
-		if b.Attestation == nil && b.Operation == nil {
-			return true
-		}
-		return false
-	}
-}
-
 type Sequence []Instruction
 
 func (seq Sequence) Compute(initial []byte) []byte {
@@ -111,11 +82,11 @@ func (seq Sequence) Compute(initial []byte) []byte {
 	return current
 }
 
-func (ts Timestamp) GetPendingSequences() []Sequence {
+func (ts File) GetPendingSequences() []Sequence {
 	bitcoin := ts.GetBitcoinAttestedSequences()
 
-	results := make([]Sequence, 0, len(ts.Instructions))
-	for _, seq := range ts.Instructions {
+	results := make([]Sequence, 0, len(ts.Sequences))
+	for _, seq := range ts.Sequences {
 		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.CalendarServerURL != "" {
 			// this is a calendar sequence, fine
 			// now we check if this same sequence isn't contained in a bigger one that contains a bitcoin attestation
@@ -125,7 +96,7 @@ func (ts Timestamp) GetPendingSequences() []Sequence {
 					continue
 				}
 
-				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return a.Equal(b) }) {
+				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return CompareInstructions(a, b) == 0 }) {
 					goto thisSequenceIsAlreadyConfirmed
 				}
 			}
@@ -141,9 +112,9 @@ func (ts Timestamp) GetPendingSequences() []Sequence {
 	return results
 }
 
-func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence {
-	results := make([]Sequence, 0, len(ts.Instructions))
-	for _, seq := range ts.Instructions {
+func (ts File) GetBitcoinAttestedSequences() []Sequence {
+	results := make([]Sequence, 0, len(ts.Sequences))
+	for _, seq := range ts.Sequences {
 		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.BitcoinBlockHeight > 0 {
 			results = append(results, seq)
 		}
@@ -151,12 +122,12 @@ func (ts Timestamp) GetBitcoinAttestedSequences() []Sequence {
 	return results
 }
 
-func (ts Timestamp) Human() string {
+func (ts File) Human() string {
 	strs := make([]string, 0, 100)
 	strs = append(strs, fmt.Sprintf("file digest: %x", ts.Digest))
 	strs = append(strs, fmt.Sprintf("hashed with: sha256"))
 	strs = append(strs, "instruction sequences:")
-	for _, seq := range ts.Instructions {
+	for _, seq := range ts.Sequences {
 		strs = append(strs, "~>")
 		for _, inst := range seq {
 			line := "  "
@@ -176,7 +147,7 @@ func (ts Timestamp) Human() string {
 	return strings.Join(strs, "\n")
 }
 
-func (ts Timestamp) SerializeToFile() []byte {
+func (ts File) SerializeToFile() []byte {
 	data := make([]byte, 0, 5050)
 	data = append(data, headerMagic...)
 	data = appendVarUint(data, 1)
@@ -186,43 +157,77 @@ func (ts Timestamp) SerializeToFile() []byte {
 	return data
 }
 
-func (ts Timestamp) SerializeInstructionSequences() []byte {
-	data := make([]byte, 0, 5000)
-	for i, seq := range ts.Instructions {
-		for _, inst := range seq {
+func (ts File) SerializeInstructionSequences() []byte {
+	sequences := make([]Sequence, len(ts.Sequences))
+	copy(sequences, ts.Sequences)
+
+	// first we sort everything so the checkpoint stuff makes sense
+	slices.SortFunc(sequences, func(a, b Sequence) int { return slices.CompareFunc(a, b, CompareInstructions) })
+
+	// checkpoints we may leave to the next people
+	sequenceCheckpoints := make([][]int, len(sequences))
+	for s1 := range sequences {
+		// keep an ordered slice of all the checkpoints we will potentially leave during our write journey for this sequence
+		checkpoints := make([]int, 0, len(sequences[s1]))
+		for s2 := s1 + 1; s2 < len(sequences); s2++ {
+			chp := getCommonPrefixIndex(sequences[s1], sequences[s2])
+			if pos, found := slices.BinarySearch(checkpoints, chp); !found {
+				checkpoints = append(checkpoints, -1)        // make room
+				copy(checkpoints[pos+1:], checkpoints[pos:]) // move elements to the right
+				checkpoints[pos] = chp                       // insert this
+			}
+		}
+		sequenceCheckpoints[s1] = checkpoints
+	}
+
+	// now actually go through the sequences writing them
+	result := make([]byte, 0, 500)
+	for s, seq := range sequences {
+		startingAt := 0
+		if s > 0 {
+			// we will always start at the last checkpoint left by the previous sequence
+			startingAt = sequenceCheckpoints[s-1][len(sequenceCheckpoints[s-1])-1]
+		}
+
+		for i := startingAt; i < len(seq); i++ {
+			// before writing anything, decide if we wanna leave a checkpoint here
+			for _, chk := range sequenceCheckpoints[s] {
+				if chk == i {
+					// leave a checkpoint
+					result = append(result, 0xff)
+				}
+			}
+
+			inst := seq[i]
 			if inst.Operation != nil {
 				// write normal operation
-				data = append(data, inst.Operation.Tag)
+				result = append(result, inst.Operation.Tag)
 				if inst.Operation.Binary {
-					data = appendVarBytes(data, inst.Argument)
+					result = appendVarBytes(result, inst.Argument)
 				}
 			} else if inst.Attestation != nil {
 				// write attestation record
-				data = append(data, 0x00)
+				result = append(result, 0x00)
 				{
-					// will use a new buffer for the actual attestation data
+					// will use a new buffer for the actual attestation result
 					abuf := make([]byte, 0, 100)
 					if inst.BitcoinBlockHeight != 0 {
-						data = append(data, bitcoinMagic...) // this goes in the main data buffer
+						result = append(result, bitcoinMagic...) // this goes in the main result buffer
 						abuf = appendVarUint(abuf, inst.BitcoinBlockHeight)
 					} else if inst.CalendarServerURL != "" {
-						data = append(data, pendingMagic...) // this goes in the main data buffer
+						result = append(result, pendingMagic...) // this goes in the main result buffer
 						abuf = appendVarBytes(abuf, []byte(inst.CalendarServerURL))
 					} else {
 						panic(fmt.Sprintf("invalid attestation: %v", inst))
 					}
-					data = appendVarBytes(data, abuf) // we append that data as varbytes
+					result = appendVarBytes(result, abuf) // we append that result as varbytes
 				}
 			} else {
 				panic(fmt.Sprintf("invalid instruction: %v", inst))
 			}
 		}
-		if i+1 < len(ts.Instructions) {
-			// write separator and start a new sequence of instructions
-			data = append(data, 0xff)
-		}
 	}
-	return data
+	return result
 }
 
 type Attestation struct {

parsers.go

@@ -19,7 +19,7 @@ func parseCalendarServerResponse(buf Buffer) (Sequence, error) {
 	return seqs[0], nil
 }
 
-func parseOTSFile(buf Buffer) (*Timestamp, error) {
+func parseOTSFile(buf Buffer) (*File, error) {
 	// read magic
 	// read version [1 byte]
 	// read crypto operation for file digest [1 byte]
@@ -47,14 +47,14 @@ func parseOTSFile(buf Buffer) (*Timestamp, error) {
 		return nil, fmt.Errorf("failed to read 32-byte digest: %w", err)
 	}
 
-	ts := &Timestamp{
+	ts := &File{
 		Digest: digest,
 	}
 
 	if seqs, err := parseTimestamp(buf); err != nil {
 		return nil, err
 	} else {
-		ts.Instructions = seqs
+		ts.Sequences = seqs
 	}
 
 	return ts, nil
@@ -102,7 +102,7 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) {
 			if err != nil {
 				return nil, fmt.Errorf("failed to read attestation bytes: %w", err)
 			}
-			abuf := NewBuffer(this)
+			abuf := newBuffer(this)
 
 			switch {
 			case slices.Equal(magic, pendingMagic):
@@ -131,17 +131,17 @@ func parseTimestamp(buf Buffer) ([]Sequence, error) {
 			ncheckpoints := len(checkpoints)
 			if ncheckpoints > 0 {
 				// use this checkpoint as the starting point for the next block
-				cp := checkpoints[ncheckpoints-1]
+				chp := checkpoints[ncheckpoints-1]
 				checkpoints = checkpoints[0 : ncheckpoints-1] // remove this from the stack
-				seqs = append(seqs, cp)
+				seqs = append(seqs, chp)
 				currInstructionsBlock++
 			}
 		} else if tag == 0xff {
 			// pick up a checkpoint to be used later
 			currentBlock := seqs[currInstructionsBlock]
-			cp := make([]Instruction, len(currentBlock))
-			copy(cp, currentBlock)
-			checkpoints = append(checkpoints, cp)
+			chp := make([]Instruction, len(currentBlock))
+			copy(chp, currentBlock)
+			checkpoints = append(checkpoints, chp)
 		} else {
 			// a new operation in this block
 			inst, err := readInstruction(buf, tag)

stamp.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 )
 
-func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp, error) {
+func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (Sequence, error) {
 	body := bytes.NewBuffer(digest[:])
 	req, err := http.NewRequestWithContext(ctx, "POST", normalizeUrl(calendarUrl)+"/digest", body)
 	if err != nil {
@@ -29,19 +29,16 @@ func Stamp(ctx context.Context, calendarUrl string, digest [32]byte) (*Timestamp
 	}
 	resp.Body.Close()
 
-	seq, err := parseCalendarServerResponse(NewBuffer(full))
+	seq, err := parseCalendarServerResponse(newBuffer(full))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response from '%s': %w", calendarUrl, err)
 	}
 
-	return &Timestamp{
-		Digest:       digest[:],
-		Instructions: []Sequence{seq},
-	}, nil
+	return seq, nil
 }
 
-func ReadFromFile(data []byte) (*Timestamp, error) {
-	return parseOTSFile(NewBuffer(data))
+func ReadFromFile(data []byte) (*File, error) {
+	return parseOTSFile(newBuffer(data))
 }
 
 func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, error) {
@@ -72,7 +69,7 @@ func (seq Sequence) Upgrade(ctx context.Context, initial []byte) (Sequence, erro
 	}
 	resp.Body.Close()
 
-	newSeq, err := parseCalendarServerResponse(NewBuffer(body))
+	newSeq, err := parseCalendarServerResponse(newBuffer(body))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response from '%s': %w", attestation.CalendarServerURL, err)
 	}

utils.go

@@ -20,7 +20,7 @@ type Buffer struct {
 	buf []byte
 }
 
-func NewBuffer(buf []byte) Buffer {
+func newBuffer(buf []byte) Buffer {
 	zero := 0
 	return Buffer{&zero, buf}
 }
@@ -99,3 +99,13 @@ func appendVarBytes(buf []byte, value []byte) []byte {
 	buf = append(buf, value...)
 	return buf
 }
+
+func getCommonPrefixIndex(s1 []Instruction, s2 []Instruction) int {
+	n := min(len(s1), len(s2))
+	for i := 0; i < n; i++ {
+		if CompareInstructions(s1[i], s2[i]) != 0 {
+			return i
+		}
+	}
+	return n
+}

verifier.go

@@ -36,7 +36,7 @@ func (seq Sequence) Verify(bitcoin Bitcoin, initial []byte) error {
 	merkleRoot := blockHeader.MerkleRoot[:]
 
 	result := seq.Compute(initial)
-	if slices.Equal(result, merkleRoot) {
+	if !slices.Equal(result, merkleRoot) {
 		return fmt.Errorf("sequence result '%x' doesn't match the bitcoin merkle root for block %d: %x",
 			result, att.BitcoinBlockHeight, merkleRoot)
 	}