Merge branch 'master' into rename-new-slot

data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.data.eraFileFormat;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.tuweni.bytes.Bytes;
+
+public class EraFile {
+  private final ByteBuffer byteBuffer;
+  private final RandomAccessFile file;
+  private final FileChannel channel;
+  private final long fileLength;
+
+  private ReadSlotIndex stateIndices;
+  private final String filename;
+
+  public EraFile(final Path path) throws IOException {
+    filename = path.getFileName().toString();
+    file = new RandomAccessFile(path.toFile(), "r");
+    channel = file.getChannel();
+    fileLength = channel.size();
+
+    final MappedByteBuffer mbb =
+        channel.map(
+            FileChannel.MapMode.READ_ONLY,
+            0, // position
+            file.length());
+    byteBuffer = mbb.order(ByteOrder.LITTLE_ENDIAN);
+  }
+
+  public void readEraFile() {
+    System.out.println("Reading Indices from ERA File " + filename);
+    getStateIndices();
+    getBlockIndices();
+  }
+
+  public void printStats() {
+    int offset = 0;
+    final Map<Bytes, Integer> entries = new HashMap<>();
+    final Map<Bytes, Integer> entryBytes = new HashMap<>();
+    while (offset < fileLength) {
+      ReadEntry entry = new ReadEntry(byteBuffer, offset);
+      offset += 8; // header
+      offset += (int) entry.getDataSize();
+      final Bytes key = Bytes.wrap(entry.getType());
+      int i = entries.getOrDefault(key, 0);
+      int size = entryBytes.getOrDefault(key, 0);
+      entries.put(key, i + 1);
+      entryBytes.put(key, size + (int) entry.getDataSize());
+    }
+
+    System.out.println("\nSummary Statistics");
+    for (Bytes key : entries.keySet().stream().sorted().toList()) {
+      final int count = entries.get(key);
+      final int bytes = entryBytes.get(key);
+      final double average = (double) bytes / (double) count;
+      System.out.printf(
+          "\ttype %s, bytes %10d, count %5d, average %12.02f%n",
+          key.toUnprefixedHexString(), bytes, count, average);
+    }
+  }
+
+  private void getStateIndices() {
+    stateIndices = new ReadSlotIndex(byteBuffer, (int) fileLength);
+    long recordStart = stateIndices.getRecordStart();
+    if (!stateIndices.getEntry().isIndexType()) {
+      throw new RuntimeException(
+          String.format(
+              " -- State e2store record type (i2) was not found at starting position "
+                  + recordStart));
+    }
+    System.out.println("\tState slot: " + stateIndices.getStartSlot());
+    System.out.println("\tState index start: " + recordStart);
+    System.out.println("\tOffsets: " + stateIndices.getSlotOffsets());
+  }
+
+  private void getBlockIndices() {
+    if (stateIndices.getStartSlot() > 0L) {
+      ReadSlotIndex blockIndices =
+          new ReadSlotIndex(byteBuffer, (int) stateIndices.getRecordStart());
+      System.out.println("\tCount of blocks: " + blockIndices.getCount());
+      System.out.println("\tBlock start slot: " + blockIndices.getStartSlot());
+      System.out.println("\tOffsets: " + blockIndices.getSlotOffsets().size());
+    } else {
+      System.out.println("Slot 0");
+    }
+  }
+
+  final void close() throws IOException {
+    channel.close();
+    file.close();
+  }
+
+  public static void main(String[] args) throws IOException {
+    final Path p = Path.of(args[0]);
+    final EraFile f = new EraFile(p);
+    f.readEraFile();
+    f.printStats();
+    f.close();
+  }
+}

data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/ReadEntry.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.data.eraFileFormat;
+
+import com.google.common.primitives.Longs;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+class ReadEntry {
+  private final byte[] type;
+  private final byte[] data;
+
+  static final byte[] INDEX_ENTRY_TYPE = {'i', '2'};
+  static final byte[] BLOCK_ENTRY_TYPE = {1, 0};
+  static final byte[] STATE_ENTRY_TYPE = {2, 0};
+
+  private final long dataSize;
+
+  public ReadEntry(final ByteBuffer byteBuffer, final int offset) {
+    // 8 byte header - first 2 are type, last 6 are little endian size
+    this.type = new byte[] {byteBuffer.get(offset), byteBuffer.get(offset + 1)};
+    final byte[] bsize = {
+      (byte) 0,
+      (byte) 0,
+      byteBuffer.get(offset + 7),
+      byteBuffer.get(offset + 6),
+      byteBuffer.get(offset + 5),
+      byteBuffer.get(offset + 4),
+      byteBuffer.get(offset + 3),
+      byteBuffer.get(offset + 2),
+    };
+    dataSize = Longs.fromByteArray(bsize);
+
+    data = new byte[(int) dataSize];
+    // read data from original position plus 8 into data
+    byteBuffer.slice(offset + 8, (int) dataSize).get(data);
+  }
+
+  public byte[] getType() {
+    return type;
+  }
+
+  public byte[] getData() {
+    return data;
+  }
+
+  public long getDataSize() {
+    return dataSize;
+  }
+
+  public boolean isIndexType() {
+    return Arrays.equals(getType(), INDEX_ENTRY_TYPE);
+  }
+
+  public boolean isBlockType() {
+    return Arrays.equals(getType(), BLOCK_ENTRY_TYPE);
+  }
+
+  public boolean isStateType() {
+    return Arrays.equals(getType(), STATE_ENTRY_TYPE);
+  }
+}

data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/ReadSlotIndex.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.data.eraFileFormat;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.primitives.Longs;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+class ReadSlotIndex {
+
+  private final long startSlot;
+  private final long recordStart;
+  private final List<Integer> slotOffsets = new ArrayList<>();
+  private final long recordEnd;
+  private final long count;
+  private final ReadEntry entry;
+
+  ReadSlotIndex(final ByteBuffer byteBuffer, final int offset) {
+    this.recordEnd = offset;
+    this.count = byteBuffer.getLong((int) recordEnd - 8);
+    this.recordStart = recordEnd - (8 * count + 24);
+    this.entry = new ReadEntry(byteBuffer, (int) recordStart);
+
+    final byte[] data = entry.getData();
+    this.startSlot = getBigEndianLongFromLittleEndianData(data, 0);
+
+    for (int i = 8; i < entry.getDataSize() - 15; i += 8) {
+      long relativePosition = getBigEndianLongFromLittleEndianData(data, i);
+      slotOffsets.add((int) relativePosition);
+    }
+  }
+
+  long getBigEndianLongFromLittleEndianData(final byte[] data, final int startOffset) {
+
+    checkArgument(
+        data.length >= startOffset + 8,
+        "buffer length is not sufficient to read a long from position " + startOffset);
+    return Longs.fromBytes(
+        data[startOffset + 7],
+        data[startOffset + 6],
+        data[startOffset + 5],
+        data[startOffset + 4],
+        data[startOffset + 3],
+        data[startOffset + 2],
+        data[startOffset + 1],
+        data[startOffset]);
+  }
+
+  public long getStartSlot() {
+    return startSlot;
+  }
+
+  public long getRecordStart() {
+    return recordStart;
+  }
+
+  public List<Integer> getSlotOffsets() {
+    return slotOffsets;
+  }
+
+  public long getCount() {
+    return count;
+  }
+
+  public ReadEntry getEntry() {
+    return entry;
+  }
+}

data/dataexchange/src/test/java/tech/pegasys/teku/data/eraFileFormat/ReadEntryTest.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.data.eraFileFormat;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.common.primitives.Longs;
+import java.nio.ByteBuffer;
+import org.junit.jupiter.api.Test;
+
+public class ReadEntryTest {
+  private final byte[] dataBytes = {10, 0, 0, 0, 0, 0, 0, 0};
+
+  @Test
+  void shouldReadTypeAndSizeFromBuffer() {
+    final byte[] bytes = encodeBytes(ReadEntry.INDEX_ENTRY_TYPE, dataBytes);
+    ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+    final ReadEntry entry = new ReadEntry(byteBuffer, 0);
+    assertThat(entry.isIndexType()).isTrue();
+    assertThat(entry.getDataSize()).isEqualTo(dataBytes.length);
+    assertThat(entry.getData()).isEqualTo(dataBytes);
+  }
+
+  @Test
+  void shouldDetectIncorrectType() {
+    final byte[] type = {'z', 'z'};
+    final byte[] bytes = encodeBytes(type, dataBytes);
+    ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+    final ReadEntry entry = new ReadEntry(byteBuffer, 0);
+    assertThat(entry.isIndexType()).isFalse();
+  }
+
+  @Test
+  void shouldDetectStateType() {
+    final byte[] bytes = encodeBytes(ReadEntry.STATE_ENTRY_TYPE, dataBytes);
+    ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+    final ReadEntry entry = new ReadEntry(byteBuffer, 0);
+    assertThat(entry.isStateType()).isTrue();
+    assertThat(entry.getDataSize()).isEqualTo(dataBytes.length);
+    assertThat(entry.getData()).isEqualTo(dataBytes);
+  }
+
+  private byte[] encodeBytes(final byte[] type, final byte[] dataLittleEndian) {
+    final byte[] bytes = new byte[8 + dataLittleEndian.length];
+    bytes[0] = type[0];
+    bytes[1] = type[1];
+    final byte[] sizeBytes = Longs.toByteArray(dataLittleEndian.length);
+    for (int i = 0; i < 7; i++) {
+      // big endian to little endian, and only the first 6 bytes for size
+      bytes[2 + i] = sizeBytes[7 - i];
+    }
+    System.arraycopy(dataLittleEndian, 0, bytes, 8, dataLittleEndian.length);
+    return bytes;
+  }
+}