add bitmask to encapsulate logic around header information in it. also add vanity count as part of the bitmask

Author: jonastheis

rollup/missing_header_fields/export-headers-toolkit/README.md

@@ -29,12 +29,11 @@ Where:
 - unique_vanity_i: unique vanity i
 - header_i: block header i
 - header: 
-    <vanity_index:uint8><flags:uint8><seal:[65|85]byte>
-    - vanity_index: index of the vanity in the unique vanity list
+    <flags:uint8><seal:[65|85]byte>
     - flags: bitmask, lsb first
-        - bit 0: 0 if difficulty is 2, 1 if difficulty is 1
-        - bit 1: 0 if seal length is 65, 1 if seal length is 85
-        - rest: 0
+	    - bit 0-5: index of the vanity in the sorted vanities list
+	    - bit 6: 0 if difficulty is 2, 1 if difficulty is 1
+	    - bit 7: 0 if seal length is 65, 1 if seal length is 85
 ```
 
 ## How to run

rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go

@@ -24,11 +24,10 @@ The binary layout of the deduplicated file is as follows:
 - 1 byte for the count of unique vanity
 - 32 bytes for each unique vanity
 - for each header:
-  - 1 byte for the index of the vanity in the unique vanity list
   - 1 byte (bitmask, lsb first): 
-	- bit 0: 0 if difficulty is 2, 1 if difficulty is 1
-    - bit 1: 0 if seal length is 65, 1 if seal length is 85
-    - rest: 0
+	- bit 0-5: index of the vanity in the sorted vanities list
+	- bit 6: 0 if difficulty is 2, 1 if difficulty is 1
+	- bit 7: 0 if seal length is 65, 1 if seal length is 85
   - 65 or 85 bytes for the seal`,
 	Run: func(cmd *cobra.Command, args []string) {
 		inputFile, err := cmd.Flags().GetString("input")
@@ -40,8 +39,8 @@ The binary layout of the deduplicated file is as follows:
 			log.Fatalf("Error reading output flag: %v", err)
 		}
 
-		seenDifficulty, seenVanity, seenSealLen := runAnalysis(inputFile)
-		runDedup(inputFile, outputFile, seenDifficulty, seenVanity, seenSealLen)
+		_, seenVanity, _ := runAnalysis(inputFile)
+		runDedup(inputFile, outputFile, seenVanity)
 		runSHA256(outputFile)
 	},
 }
@@ -105,7 +104,7 @@ func runAnalysis(inputFile string) (seenDifficulty map[uint64]int, seenVanity ma
 	return seenDifficulty, seenVanity, seenSealLen
 }
 
-func runDedup(inputFile, outputFile string, seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) {
+func runDedup(inputFile, outputFile string, seenVanity map[[32]byte]bool) {
 	reader := newHeaderReader(inputFile)
 	defer reader.close()
 

rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go

@@ -11,6 +11,9 @@ import (
 	"github.com/scroll-tech/go-ethereum/export-headers-toolkit/types"
 )
 
+// maxVanityCount is the maximum number of unique vanities that can be represented with 6 bits in the bitmask
+const maxVanityCount = 1 << 6
+
 type missingHeaderFileWriter struct {
 	file   *os.File
 	writer *bufio.Writer
@@ -19,6 +22,10 @@ type missingHeaderFileWriter struct {
 }
 
 func newMissingHeaderFileWriter(filename string, seenVanity map[[32]byte]bool) *missingHeaderFileWriter {
+	if len(seenVanity) > maxVanityCount {
+		log.Fatalf("Number of unique vanities exceeds maximum: %d > %d", len(seenVanity), maxVanityCount)
+	}
+
 	file, err := os.Create(filename)
 	if err != nil {
 		log.Fatalf("Error creating file: %v", err)
@@ -86,28 +93,73 @@ func (m *missingHeaderWriter) writeVanities() {
 }
 
 func (m *missingHeaderWriter) write(header *types.Header) {
-	// 1. write the index of the vanity in the unique vanity list
-	if _, err := m.writer.Write([]byte{uint8(m.sortedVanitiesMap[header.Vanity()])}); err != nil {
-		log.Fatalf("Error writing vanity index: %v", err)
+	// 1. prepare the bitmask
+	bits := newBitMask(m.sortedVanitiesMap[header.Vanity()], int(header.Difficulty), header.SealLen())
+
+	// 2. write the header: bitmask and seal
+	if _, err := m.writer.Write(bits.Bytes()); err != nil {
+		log.Fatalf("Error writing bitmask: %v", err)
 	}
 
-	// 2. write the bitmask
-	// - bit 0: 0 if difficulty is 2, 1 if difficulty is 1
-	// - bit 1: 0 if seal length is 65, 1 if seal length is 85
-	// - rest: 0
-	bitmask := uint8(0)
-	if header.Difficulty == 1 {
-		bitmask |= 1 << 0
+	if _, err := m.writer.Write(header.Seal()); err != nil {
+		log.Fatalf("Error writing seal: %v", err)
 	}
-	if header.SealLen() == 85 {
-		bitmask |= 1 << 1
+}
+
+// bitMask is a bitmask that encodes the following information:
+//
+//	bit 0-5: index of the vanity in the sorted vanities list
+//	bit 6: 0 if difficulty is 2, 1 if difficulty is 1
+//	bit 7: 0 if seal length is 65, 1 if seal length is 85
+type bitMask struct {
+	b uint8
+}
+
+func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask {
+	b := uint8(0)
+
+	if vanityIndex >= maxVanityCount {
+		log.Fatalf("Vanity index exceeds maximum: %d >= %d", vanityIndex, maxVanityCount)
 	}
+	b |= uint8(vanityIndex) & 0b00111111
 
-	if _, err := m.writer.Write([]byte{bitmask}); err != nil {
-		log.Fatalf("Error writing bitmask: %v", err)
+	if difficulty == 1 {
+		b |= 1 << 6
+	} else if difficulty != 2 {
+		log.Fatalf("Invalid difficulty: %d", difficulty)
 	}
 
-	if _, err := m.writer.Write(header.Seal()); err != nil {
-		log.Fatalf("Error writing seal: %v", err)
+	if sealLen == 85 {
+		b |= 1 << 7
+	} else if sealLen != 65 {
+		log.Fatalf("Invalid seal length: %d", sealLen)
 	}
+
+	return bitMask{b}
+}
+
+func (b bitMask) vanityIndex() int {
+	return int(b.b & 0b00111111)
+}
+
+func (b bitMask) difficulty() int {
+	val := (b.b >> 6) & 0x01
+	if val == 0 {
+		return 2
+	} else {
+		return 1
+	}
+}
+
+func (b bitMask) sealLen() int {
+	val := (b.b >> 7) & 0x01
+	if val == 0 {
+		return 65
+	} else {
+		return 85
+	}
+}
+
+func (b bitMask) Bytes() []byte {
+	return []byte{b.b}
 }

rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go

@@ -39,8 +39,8 @@ func TestMissingHeaderWriter(t *testing.T) {
 		header := types.NewHeader(0, 2, append(vanity1[:], seal...))
 		mhw.write(header)
 
-		expectedBytes = append(expectedBytes, 0x00) // index 0
-		expectedBytes = append(expectedBytes, 0x00) // difficulty 2, seal length 65
+		// bit 0-5:0x0 index0, bit 6=0: difficulty 2, bit 7=0: seal length 65
+		expectedBytes = append(expectedBytes, 0b00000000)
 		expectedBytes = append(expectedBytes, seal...)
 		assert.Equal(t, expectedBytes, bytesBuffer.Bytes())
 	}
@@ -51,8 +51,8 @@ func TestMissingHeaderWriter(t *testing.T) {
 		header := types.NewHeader(1, 1, append(vanity2[:], seal...))
 		mhw.write(header)
 
-		expectedBytes = append(expectedBytes, 0x01) // index 1
-		expectedBytes = append(expectedBytes, 0x01) // difficulty 1, seal length 65
+		// bit 0-5:0x1 index1, bit 6=1: difficulty 1, bit 7=0: seal length 65
+		expectedBytes = append(expectedBytes, 0b01000001)
 		expectedBytes = append(expectedBytes, seal...)
 		assert.Equal(t, expectedBytes, bytesBuffer.Bytes())
 	}
@@ -63,8 +63,8 @@ func TestMissingHeaderWriter(t *testing.T) {
 		header := types.NewHeader(2, 2, append(vanity2[:], seal...))
 		mhw.write(header)
 
-		expectedBytes = append(expectedBytes, 0x01) // index 1
-		expectedBytes = append(expectedBytes, 0x02) // difficulty 2, seal length 85
+		// bit 0-5:0x1 index1, bit 6=0: difficulty 2, bit 7=1: seal length 85
+		expectedBytes = append(expectedBytes, 0b10000001)
 		expectedBytes = append(expectedBytes, seal...)
 		assert.Equal(t, expectedBytes, bytesBuffer.Bytes())
 	}
@@ -75,8 +75,8 @@ func TestMissingHeaderWriter(t *testing.T) {
 		header := types.NewHeader(3, 1, append(vanity8[:], seal...))
 		mhw.write(header)
 
-		expectedBytes = append(expectedBytes, 0x02) // index 2
-		expectedBytes = append(expectedBytes, 0x03) // difficulty 1, seal length 85
+		// bit 0-5:0x2 index2, bit 6=1: difficulty 1, bit 7=1: seal length 85
+		expectedBytes = append(expectedBytes, 0b11000010)
 		expectedBytes = append(expectedBytes, seal...)
 		assert.Equal(t, expectedBytes, bytesBuffer.Bytes())
 	}
@@ -87,8 +87,8 @@ func TestMissingHeaderWriter(t *testing.T) {
 		header := types.NewHeader(4, 2, append(vanity1[:], seal...))
 		mhw.write(header)
 
-		expectedBytes = append(expectedBytes, 0x00) // index 0
-		expectedBytes = append(expectedBytes, 0x00) // difficulty 2, seal length 65
+		// bit 0-5:0x0 index0, bit 6=0: difficulty 2, bit 7=0: seal length 65
+		expectedBytes = append(expectedBytes, 0b00000000)
 		expectedBytes = append(expectedBytes, seal...)
 		assert.Equal(t, expectedBytes, bytesBuffer.Bytes())
 	}