refactor

ots/attestation.go

@@ -0,0 +1,28 @@
+package ots
+
+import "fmt"
+
+type Attestation struct {
+	BitcoinBlockHeight uint64
+	CalendarServerURL  string
+}
+
+func (att Attestation) Name() string {
+	if att.BitcoinBlockHeight != 0 {
+		return "bitcoin"
+	} else if att.CalendarServerURL != "" {
+		return "pending"
+	} else {
+		return "unknown/broken"
+	}
+}
+
+func (att Attestation) Human() string {
+	if att.BitcoinBlockHeight != 0 {
+		return fmt.Sprintf("bitcoin(%d)", att.BitcoinBlockHeight)
+	} else if att.CalendarServerURL != "" {
+		return fmt.Sprintf("pending(%s)", att.CalendarServerURL)
+	} else {
+		return "unknown/broken"
+	}
+}

ots/crypto.go

@@ -0,0 +1,9 @@
+package ots
+
+import (
+	deprecated_ripemd160 "golang.org/x/crypto/ripemd160"
+)
+
+func ripemd160(curr []byte, arg []byte) []byte {
+	return deprecated_ripemd160.New().Sum(curr)
+}

ots/file.go

@@ -0,0 +1,222 @@
+package ots
+
+import (
+	"encoding/hex"
+	"fmt"
+	"slices"
+	"strings"
+
+	"git.intruders.space/public/opentimestamps/varn"
+)
+
+// Header magic bytes
+// Designed to be give the user some information in a hexdump, while being identified as 'data' by the file utility.
+// \x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94
+var (
+	HeaderMagic  = []byte{0x00, 0x4f, 0x70, 0x65, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x73, 0x00, 0x00, 0x50, 0x72, 0x6f, 0x6f, 0x66, 0x00, 0xbf, 0x89, 0xe2, 0xe8, 0x84, 0xe8, 0x92, 0x94}
+	PendingMagic = []byte{0x83, 0xdf, 0xe3, 0x0d, 0x2e, 0xf9, 0x0c, 0x8e}
+	BitcoinMagic = []byte{0x05, 0x88, 0x96, 0x0d, 0x73, 0xd7, 0x19, 0x01}
+)
+
+// A File represents the parsed content of an .ots file: it has an initial digest and
+// a series of sequences of instructions. Each sequence must be evaluated separately, applying the operations
+// on top of each other, starting with the .Digest until they end on an attestation.
+type File struct {
+	Digest    []byte
+	Sequences []Sequence
+}
+
+func (ts File) GetPendingSequences() []Sequence {
+	bitcoin := ts.GetBitcoinAttestedSequences()
+
+	results := make([]Sequence, 0, len(ts.Sequences))
+	for _, seq := range ts.Sequences {
+		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.CalendarServerURL != "" {
+			// this is a calendar sequence, fine
+			// now we check if this same sequence isn't contained in a bigger one that contains a bitcoin attestation
+			cseq := seq
+			for _, bseq := range bitcoin {
+				if len(bseq) < len(cseq) {
+					continue
+				}
+
+				if slices.EqualFunc(bseq[0:len(cseq)], cseq, func(a, b Instruction) bool { return CompareInstructions(a, b) == 0 }) {
+					goto thisSequenceIsAlreadyConfirmed
+				}
+			}
+
+			// sequence not confirmed, so add it to pending result
+			results = append(results, seq)
+
+		thisSequenceIsAlreadyConfirmed:
+			// skip this
+			continue
+		}
+	}
+	return results
+}
+
+func (ts File) GetBitcoinAttestedSequences() []Sequence {
+	results := make([]Sequence, 0, len(ts.Sequences))
+	for _, seq := range ts.Sequences {
+		if len(seq) > 0 && seq[len(seq)-1].Attestation != nil && seq[len(seq)-1].Attestation.BitcoinBlockHeight > 0 {
+			results = append(results, seq)
+		}
+	}
+	return results
+}
+
+func (ts File) Human(withPartials bool) string {
+	strs := make([]string, 0, 100)
+	strs = append(strs, fmt.Sprintf("file digest: %x", ts.Digest))
+	strs = append(strs, fmt.Sprintf("hashed with: sha256"))
+	strs = append(strs, "instruction sequences:")
+	for _, seq := range ts.Sequences {
+		curr := ts.Digest
+		strs = append(strs, "~>")
+		strs = append(strs, "  start "+hex.EncodeToString(curr))
+		for _, inst := range seq {
+			line := "  "
+			if inst.Operation != nil {
+				line += inst.Operation.Name
+				curr = inst.Operation.Apply(curr, inst.Argument)
+				if inst.Operation.Binary {
+					line += " " + hex.EncodeToString(inst.Argument)
+				}
+				if withPartials {
+					line += " = " + hex.EncodeToString(curr)
+				}
+			} else if inst.Attestation != nil {
+				line += inst.Attestation.Human()
+			} else {
+				panic(fmt.Sprintf("invalid instruction timestamp: %v", inst))
+			}
+			strs = append(strs, line)
+		}
+	}
+	return strings.Join(strs, "\n")
+}
+
+func (ts File) SerializeToFile() []byte {
+	data := make([]byte, 0, 5050)
+	data = append(data, HeaderMagic...)
+	data = varn.AppendVarUint(data, 1)
+	data = append(data, 0x08) // sha256
+	data = append(data, ts.Digest...)
+	data = append(data, ts.SerializeInstructionSequences()...)
+	return data
+}
+
+func (ts File) SerializeInstructionSequences() []byte {
+	sequences := make([]Sequence, len(ts.Sequences))
+	copy(sequences, ts.Sequences)
+
+	// first we sort everything so the checkpoint stuff makes sense
+	slices.SortFunc(sequences, func(a, b Sequence) int { return slices.CompareFunc(a, b, CompareInstructions) })
+
+	// checkpoints we may leave to the next people
+	sequenceCheckpoints := make([][]int, len(sequences))
+	for s1 := range sequences {
+		// keep an ordered slice of all the checkpoints we will potentially leave during our write journey for this sequence
+		checkpoints := make([]int, 0, len(sequences[s1]))
+		for s2 := s1 + 1; s2 < len(sequences); s2++ {
+			chp := GetCommonPrefixIndex(sequences[s1], sequences[s2])
+			if pos, found := slices.BinarySearch(checkpoints, chp); !found {
+				checkpoints = append(checkpoints, -1)        // make room
+				copy(checkpoints[pos+1:], checkpoints[pos:]) // move elements to the right
+				checkpoints[pos] = chp                       // insert this
+			}
+		}
+		sequenceCheckpoints[s1] = checkpoints
+	}
+
+	// now actually go through the sequences writing them
+	result := make([]byte, 0, 500)
+	for s, seq := range sequences {
+		startingAt := 0
+		if s > 0 {
+			// we will always start at the last checkpoint left by the previous sequence
+			startingAt = sequenceCheckpoints[s-1][len(sequenceCheckpoints[s-1])-1]
+		}
+
+		for i := startingAt; i < len(seq); i++ {
+			// before writing anything, decide if we wanna leave a checkpoint here
+			for _, chk := range sequenceCheckpoints[s] {
+				if chk == i {
+					// leave a checkpoint
+					result = append(result, 0xff)
+				}
+			}
+
+			inst := seq[i]
+			if inst.Operation != nil {
+				// write normal operation
+				result = append(result, inst.Operation.Tag)
+				if inst.Operation.Binary {
+					result = varn.AppendVarBytes(result, inst.Argument)
+				}
+			} else if inst.Attestation != nil {
+				// write attestation record
+				result = append(result, 0x00)
+				{
+					// will use a new buffer for the actual attestation result
+					abuf := make([]byte, 0, 100)
+					if inst.BitcoinBlockHeight != 0 {
+						result = append(result, BitcoinMagic...) // this goes in the main result buffer
+						abuf = varn.AppendVarUint(abuf, inst.BitcoinBlockHeight)
+					} else if inst.CalendarServerURL != "" {
+						result = append(result, PendingMagic...) // this goes in the main result buffer
+						abuf = varn.AppendVarBytes(abuf, []byte(inst.CalendarServerURL))
+					} else {
+						panic(fmt.Sprintf("invalid attestation: %v", inst))
+					}
+					result = varn.AppendVarBytes(result, abuf) // we append that result as varbytes
+				}
+			} else {
+				panic(fmt.Sprintf("invalid instruction: %v", inst))
+			}
+		}
+	}
+	return result
+}
+
+func ParseOTSFile(buf varn.Buffer) (*File, error) {
+	// read magic
+	// read version [1 byte]
+	// read crypto operation for file digest [1 byte]
+	// read file digest [32 byte (depends)]
+	if magic, err := buf.ReadBytes(len(HeaderMagic)); err != nil || !slices.Equal(HeaderMagic, magic) {
+		return nil, fmt.Errorf("invalid ots file header '%s': %w", magic, err)
+	}
+
+	if version, err := buf.ReadVarUint(); err != nil || version != 1 {
+		return nil, fmt.Errorf("invalid ots file version '%v': %w", version, err)
+	}
+
+	tag, err := buf.ReadByte()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read operation byte: %w", err)
+	}
+
+	if op, err := ReadInstruction(buf, tag); err != nil || op.Operation.Name != "sha256" {
+		return nil, fmt.Errorf("invalid crypto operation '%v', only sha256 supported: %w", op, err)
+	}
+
+	// if we got here assume the digest is sha256
+	digest, err := buf.ReadBytes(32)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read 32-byte digest: %w", err)
+	}
+
+	ts := &File{
+		Digest: digest,
+	}
+
+	if seqs, err := ParseTimestamp(buf); err != nil {
+		return nil, err
+	} else {
+		ts.Sequences = seqs
+	}
+
+	return ts, nil
+}

ots/instruction.go

@@ -0,0 +1,94 @@
+package ots
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"git.intruders.space/public/opentimestamps/varn"
+)
+
+// a Instruction can be an operation like "append" or "prepend" (this will be the case when .Operation != nil)
+// or an attestation (when .Attestation != nil).
+// It will have a non-nil .Argument whenever the operation requires an argument.
+type Instruction struct {
+	*Operation
+	Argument []byte
+	*Attestation
+}
+
+func GetCommonPrefixIndex(s1 []Instruction, s2 []Instruction) int {
+	n := min(len(s1), len(s2))
+	for i := 0; i < n; i++ {
+		if CompareInstructions(s1[i], s2[i]) != 0 {
+			return i
+		}
+	}
+	return n
+}
+
+// CompareInstructions returns negative if a<b, 0 if a=b and positive if a>b.
+// It considers an operation smaller than an attestation, a pending attestation smaller than a Bitcoin attestation.
+// It orders operations by their tag byte and then by their argument.
+func CompareInstructions(a, b Instruction) int {
+	if a.Operation != nil {
+		if b.Attestation != nil {
+			// a is an operation but b is an attestation, a is bigger
+			return +1
+		}
+		if a.Operation == b.Operation {
+			// if both are the same operation sort by the argument
+			return slices.Compare(a.Argument, b.Argument)
+		}
+
+		// sort by the operation
+		if a.Operation.Tag < b.Operation.Tag {
+			return -1
+		} else if a.Operation.Tag > b.Operation.Tag {
+			return 1
+		} else {
+			return 0
+		}
+	} else if a.Attestation != nil && b.Attestation == nil {
+		// a is an attestation but b is not, b is bigger
+		return -1
+	} else if a.Attestation != nil && b.Attestation != nil {
+		// both are attestations
+		if a.Attestation.BitcoinBlockHeight == 0 && b.Attestation.BitcoinBlockHeight == 0 {
+			// none are bitcoin attestations
+			return strings.Compare(a.Attestation.CalendarServerURL, b.Attestation.CalendarServerURL)
+		}
+		if a.Attestation.BitcoinBlockHeight != 0 && b.Attestation.BitcoinBlockHeight != 0 {
+			// both are bitcoin attestations
+			return int(b.Attestation.BitcoinBlockHeight - a.Attestation.BitcoinBlockHeight)
+		}
+
+		// one is bitcoin and the other is not -- compare by bitcoin block,
+		// but reverse the result since the one with 0 should not be considered bigger
+		return -1 * int(b.Attestation.BitcoinBlockHeight-a.Attestation.BitcoinBlockHeight)
+	} else {
+		// this shouldn't happen
+		return 0
+	}
+}
+
+func ReadInstruction(buf varn.Buffer, tag byte) (*Instruction, error) {
+	op, ok := Tags[tag]
+	if !ok {
+		return nil, fmt.Errorf("unknown tag %v", tag)
+	}
+
+	inst := Instruction{
+		Operation: op,
+	}
+
+	if op.Binary {
+		val, err := buf.ReadVarBytes()
+		if err != nil {
+			return nil, fmt.Errorf("error reading argument: %w", err)
+		}
+		inst.Argument = val
+	}
+
+	return &inst, nil
+}

ots/operation.go

@@ -0,0 +1,34 @@
+package ots
+
+import "crypto/sha256"
+
+type Operation struct {
+	Name   string
+	Tag    byte
+	Binary bool // it's an operation that takes one argument, otherwise takes none
+	Apply  func(curr []byte, arg []byte) []byte
+}
+
+var Tags = map[byte]*Operation{
+	0xf0: {"append", 0xf0, true, func(curr []byte, arg []byte) []byte {
+		result := make([]byte, len(curr)+len(arg))
+		copy(result[0:], curr)
+		copy(result[len(curr):], arg)
+		return result
+	}},
+	0xf1: {"prepend", 0xf1, true, func(curr []byte, arg []byte) []byte {
+		result := make([]byte, len(curr)+len(arg))
+		copy(result[0:], arg)
+		copy(result[len(arg):], curr)
+		return result
+	}},
+	0xf2: {"reverse", 0xf2, false, func(curr []byte, arg []byte) []byte { panic("reverse not implemented") }},
+	0xf3: {"hexlify", 0xf3, false, func(curr []byte, arg []byte) []byte { panic("hexlify not implemented") }},
+	0x02: {"sha1", 0x02, false, func(curr []byte, arg []byte) []byte { panic("sha1 not implemented") }},
+	0x03: {"ripemd160", 0x03, false, ripemd160},
+	0x08: {"sha256", 0x08, false, func(curr []byte, arg []byte) []byte {
+		v := sha256.Sum256(curr)
+		return v[:]
+	}},
+	0x67: {"keccak256", 0x67, false, func(curr []byte, arg []byte) []byte { panic("keccak256 not implemented") }},
+}

ots/sequence.go

@@ -0,0 +1,176 @@
+package ots
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"slices"
+
+	"git.intruders.space/public/opentimestamps/varn"
+	"git.intruders.space/public/opentimestamps/verifyer"
+	"github.com/btcsuite/btcd/wire"
+)
+
+type Sequence []Instruction
+
+func (seq Sequence) GetAttestation() Attestation {
+	if len(seq) == 0 {
+		return Attestation{}
+	}
+	att := seq[len(seq)-1]
+	if att.Attestation == nil {
+		return Attestation{}
+	}
+	return *att.Attestation
+}
+
+// Compute runs a sequence of operations on top of an initial digest and returns the result, which is often a
+// Bitcoin block merkle root. It also tries to identify the point in the sequence in which an actual Bitcoin
+// transaction is formed and parse that.
+func (seq Sequence) Compute(initial []byte) (merkleRoot []byte, bitcoinTx *wire.MsgTx) {
+	current := initial
+	for i, inst := range seq {
+		if inst.Operation == nil {
+			break
+		}
+
+		// the first time we do a double-sha256 that is likely a bitcoin transaction
+		if bitcoinTx == nil &&
+			inst.Operation.Name == "sha256" &&
+			len(seq) > i+1 && seq[i+1].Operation != nil &&
+			seq[i+1].Operation.Name == "sha256" {
+			tx := &wire.MsgTx{}
+			tx.Deserialize(bytes.NewReader(current))
+			bitcoinTx = tx
+		}
+
+		current = inst.Operation.Apply(current, inst.Argument)
+	}
+	return current, bitcoinTx
+}
+
+// Verify validates sequence of operations that starts with digest and ends on a Bitcoin attestation against
+// an actual Bitcoin block, as given by the provided Bitcoin interface.
+func (seq Sequence) Verify(bitcoin verifyer.Bitcoin, digest []byte) (*wire.MsgTx, error) {
+	if len(seq) == 0 {
+		return nil, fmt.Errorf("empty sequence")
+	}
+
+	att := seq[len(seq)-1]
+	if att.Attestation == nil || att.BitcoinBlockHeight == 0 {
+		return nil, fmt.Errorf("sequence doesn't include a bitcoin attestation")
+	}
+
+	blockHash, err := bitcoin.GetBlockHash(int64(att.BitcoinBlockHeight))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get block %d hash: %w", att.BitcoinBlockHeight, err)
+	}
+
+	blockHeader, err := bitcoin.GetBlockHeader(blockHash)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get block %s header: %w", blockHash, err)
+	}
+
+	merkleRoot := blockHeader.MerkleRoot[:]
+	result, tx := seq.Compute(digest)
+
+	if !bytes.Equal(result, merkleRoot) {
+		return nil, fmt.Errorf("sequence result '%x' doesn't match the bitcoin merkle root for block %d: %x",
+			result, att.BitcoinBlockHeight, merkleRoot)
+	}
+
+	return tx, nil
+}
+
+func ParseTimestamp(buf varn.Buffer) ([]Sequence, error) {
+	// read instructions
+	//   if operation = push
+	//   if 0x00 = attestation
+	//      read tag [8 bytes]
+	//      readvarbytes
+	//        interpret these depending on the type of attestation
+	//          if bitcoin: readvaruint as the block height
+	//          if pending from calendar: readvarbytes as the utf-8 calendar url
+	//      end or go back to last continuation byte
+	//   if 0xff = pick up a continuation byte (checkpoint) and add it to stack
+
+	currInstructionsBlock := 0
+	seqs := make([]Sequence, 0, 10)
+
+	// we will store checkpoints here
+	checkpoints := make([][]Instruction, 0, 4)
+
+	// start first instruction block
+	seqs = append(seqs, make([]Instruction, 0, 30))
+
+	// go read these tags
+	for {
+		tag, err := buf.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return seqs, nil
+			}
+			return nil, fmt.Errorf("failed to read operation byte: %w", err)
+		}
+
+		if tag == 0x00 {
+			// enter an attestation context
+			magic, err := buf.ReadBytes(8)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read attestion magic bytes: %w", err)
+			}
+
+			this, err := buf.ReadVarBytes()
+			if err != nil {
+				return nil, fmt.Errorf("failed to read attestation bytes: %w", err)
+			}
+			abuf := varn.NewBuffer(this)
+
+			switch {
+			case slices.Equal(magic, PendingMagic):
+				val, err := abuf.ReadVarBytes()
+				if err != nil {
+					return nil, fmt.Errorf("failed reading calendar server url: %w", err)
+				}
+				seqs[currInstructionsBlock] = append(
+					seqs[currInstructionsBlock],
+					Instruction{Attestation: &Attestation{CalendarServerURL: string(val)}},
+				)
+			case slices.Equal(magic, BitcoinMagic):
+				val, err := abuf.ReadVarUint()
+				if err != nil {
+					return nil, fmt.Errorf("failed reading bitcoin block number: %w", err)
+				}
+				seqs[currInstructionsBlock] = append(
+					seqs[currInstructionsBlock],
+					Instruction{Attestation: &Attestation{BitcoinBlockHeight: val}},
+				)
+			default:
+				return nil, fmt.Errorf("unsupported attestation type '%x': %x", magic, this)
+			}
+
+			// check if we have checkpoints and, if yes, copy them in a new block of instructions
+			ncheckpoints := len(checkpoints)
+			if ncheckpoints > 0 {
+				// use this checkpoint as the starting point for the next block
+				chp := checkpoints[ncheckpoints-1]
+				checkpoints = checkpoints[0 : ncheckpoints-1] // remove this from the stack
+				seqs = append(seqs, chp)
+				currInstructionsBlock++
+			}
+		} else if tag == 0xff {
+			// pick up a checkpoint to be used later
+			currentBlock := seqs[currInstructionsBlock]
+			chp := make([]Instruction, len(currentBlock))
+			copy(chp, currentBlock)
+			checkpoints = append(checkpoints, chp)
+		} else {
+			// a new operation in this block
+			inst, err := ReadInstruction(buf, tag)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read instruction: %w", err)
+			}
+			seqs[currInstructionsBlock] = append(seqs[currInstructionsBlock], *inst)
+		}
+	}
+}