/root/bitcoin/src/wallet/migrate.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // Distributed under the MIT software license, see the accompanying |
2 | | // file COPYING or http://www.opensource.org/licenses/mit-license.php. |
3 | | |
4 | | #include <compat/byteswap.h> |
5 | | #include <crypto/common.h> // For ReadBE32 |
6 | | #include <logging.h> |
7 | | #include <streams.h> |
8 | | #include <util/translation.h> |
9 | | #include <wallet/migrate.h> |
10 | | |
11 | | #include <optional> |
12 | | #include <variant> |
13 | | |
14 | | namespace wallet { |
15 | | // Magic bytes in both endianness's |
16 | | constexpr uint32_t BTREE_MAGIC = 0x00053162; // If the file endianness matches our system, we see this magic |
17 | | constexpr uint32_t BTREE_MAGIC_OE = 0x62310500; // If the file endianness is the other one, we will see this magic |
18 | | |
19 | | // Subdatabase name |
20 | | static const std::vector<std::byte> SUBDATABASE_NAME = {std::byte{'m'}, std::byte{'a'}, std::byte{'i'}, std::byte{'n'}}; |
21 | | |
22 | | enum class PageType : uint8_t { |
23 | | /* |
24 | | * BDB has several page types, most of which we do not use |
25 | | * They are listed here for completeness, but commented out |
26 | | * to avoid opening something unintended. |
27 | | INVALID = 0, // Invalid page type |
28 | | DUPLICATE = 1, // Duplicate. Deprecated and no longer used |
29 | | HASH_UNSORTED = 2, // Hash pages. Deprecated. |
30 | | RECNO_INTERNAL = 4, // Recno internal |
31 | | RECNO_LEAF = 6, // Recno leaf |
32 | | HASH_META = 8, // Hash metadata |
33 | | QUEUE_META = 10, // Queue Metadata |
34 | | QUEUE_DATA = 11, // Queue Data |
35 | | DUPLICATE_LEAF = 12, // Off-page duplicate leaf |
36 | | HASH_SORTED = 13, // Sorted hash page |
37 | | */ |
38 | | BTREE_INTERNAL = 3, // BTree internal |
39 | | BTREE_LEAF = 5, // BTree leaf |
40 | | OVERFLOW_DATA = 7, // Overflow |
41 | | BTREE_META = 9, // BTree metadata |
42 | | }; |
43 | | |
44 | | enum class RecordType : uint8_t { |
45 | | KEYDATA = 1, |
46 | | // DUPLICATE = 2, Unused as our databases do not support duplicate records |
47 | | OVERFLOW_DATA = 3, |
48 | | DELETE = 0x80, // Indicate this record is deleted. This is OR'd with the real type. |
49 | | }; |
50 | | |
51 | | enum class BTreeFlags : uint32_t { |
52 | | /* |
53 | | * BTree databases have feature flags, but we do not use them except for |
54 | | * subdatabases. The unused flags are included for completeness, but commented out |
55 | | * to avoid accidental use. |
56 | | DUP = 1, // Duplicates |
57 | | RECNO = 2, // Recno tree |
58 | | RECNUM = 4, // BTree: Maintain record counts |
59 | | FIXEDLEN = 8, // Recno: fixed length records |
60 | | RENUMBER = 0x10, // Recno: renumber on insert/delete |
61 | | DUPSORT = 0x40, // Duplicates are sorted |
62 | | COMPRESS = 0x80, // Compressed |
63 | | */ |
64 | | SUBDB = 0x20, // Subdatabases |
65 | | }; |
66 | | |
67 | | /** Berkeley DB BTree metadata page layout */ |
68 | | class MetaPage |
69 | | { |
70 | | public: |
71 | | uint32_t lsn_file; // Log Sequence Number file |
72 | | uint32_t lsn_offset; // Log Sequence Number offset |
73 | | uint32_t page_num; // Current page number |
74 | | uint32_t magic; // Magic number |
75 | | uint32_t version; // Version |
76 | | uint32_t pagesize; // Page size |
77 | | uint8_t encrypt_algo; // Encryption algorithm |
78 | | PageType type; // Page type |
79 | | uint8_t metaflags; // Meta-only flags |
80 | | uint8_t unused1; // Unused |
81 | | uint32_t free_list; // Free list page number |
82 | | uint32_t last_page; // Page number of last page in db |
83 | | uint32_t partitions; // Number of partitions |
84 | | uint32_t key_count; // Cached key count |
85 | | uint32_t record_count; // Cached record count |
86 | | BTreeFlags flags; // Flags |
87 | | std::array<std::byte, 20> uid; // 20 byte unique file ID |
88 | | uint32_t unused2; // Unused |
89 | | uint32_t minkey; // Minimum key |
90 | | uint32_t re_len; // Recno: fixed length record length |
91 | | uint32_t re_pad; // Recno: fixed length record pad |
92 | | uint32_t root; // Root page number |
93 | | char unused3[368]; // 92 * 4 bytes of unused space |
94 | | uint32_t crypto_magic; // Crypto magic number |
95 | | char trash[12]; // 3 * 4 bytes of trash space |
96 | | unsigned char iv[20]; // Crypto IV |
97 | | unsigned char chksum[16]; // Checksum |
98 | | |
99 | | bool other_endian; |
100 | | uint32_t expected_page_num; |
101 | | |
102 | 0 | MetaPage(uint32_t expected_page_num) : expected_page_num(expected_page_num) {} |
103 | | MetaPage() = delete; |
104 | | |
105 | | template <typename Stream> |
106 | | void Unserialize(Stream& s) |
107 | 0 | { |
108 | 0 | s >> lsn_file; |
109 | 0 | s >> lsn_offset; |
110 | 0 | s >> page_num; |
111 | 0 | s >> magic; |
112 | 0 | s >> version; |
113 | 0 | s >> pagesize; |
114 | 0 | s >> encrypt_algo; |
115 | |
|
116 | 0 | other_endian = magic == BTREE_MAGIC_OE; |
117 | |
|
118 | 0 | uint8_t uint8_type; |
119 | 0 | s >> uint8_type; |
120 | 0 | type = static_cast<PageType>(uint8_type); |
121 | |
|
122 | 0 | s >> metaflags; |
123 | 0 | s >> unused1; |
124 | 0 | s >> free_list; |
125 | 0 | s >> last_page; |
126 | 0 | s >> partitions; |
127 | 0 | s >> key_count; |
128 | 0 | s >> record_count; |
129 | |
|
130 | 0 | uint32_t uint32_flags; |
131 | 0 | s >> uint32_flags; |
132 | 0 | if (other_endian) { |
133 | 0 | uint32_flags = internal_bswap_32(uint32_flags); |
134 | 0 | } |
135 | 0 | flags = static_cast<BTreeFlags>(uint32_flags); |
136 | |
|
137 | 0 | s >> uid; |
138 | 0 | s >> unused2; |
139 | 0 | s >> minkey; |
140 | 0 | s >> re_len; |
141 | 0 | s >> re_pad; |
142 | 0 | s >> root; |
143 | 0 | s >> unused3; |
144 | 0 | s >> crypto_magic; |
145 | 0 | s >> trash; |
146 | 0 | s >> iv; |
147 | 0 | s >> chksum; |
148 | |
|
149 | 0 | if (other_endian) { |
150 | 0 | lsn_file = internal_bswap_32(lsn_file); |
151 | 0 | lsn_offset = internal_bswap_32(lsn_offset); |
152 | 0 | page_num = internal_bswap_32(page_num); |
153 | 0 | magic = internal_bswap_32(magic); |
154 | 0 | version = internal_bswap_32(version); |
155 | 0 | pagesize = internal_bswap_32(pagesize); |
156 | 0 | free_list = internal_bswap_32(free_list); |
157 | 0 | last_page = internal_bswap_32(last_page); |
158 | 0 | partitions = internal_bswap_32(partitions); |
159 | 0 | key_count = internal_bswap_32(key_count); |
160 | 0 | record_count = internal_bswap_32(record_count); |
161 | 0 | unused2 = internal_bswap_32(unused2); |
162 | 0 | minkey = internal_bswap_32(minkey); |
163 | 0 | re_len = internal_bswap_32(re_len); |
164 | 0 | re_pad = internal_bswap_32(re_pad); |
165 | 0 | root = internal_bswap_32(root); |
166 | 0 | crypto_magic = internal_bswap_32(crypto_magic); |
167 | 0 | } |
168 | | |
169 | | // Page number must match |
170 | 0 | if (page_num != expected_page_num) { |
171 | 0 | throw std::runtime_error("Meta page number mismatch"); |
172 | 0 | } |
173 | | |
174 | | // Check magic |
175 | 0 | if (magic != BTREE_MAGIC) { |
176 | 0 | throw std::runtime_error("Not a BDB file"); |
177 | 0 | } |
178 | | |
179 | | // Only version 9 is supported |
180 | 0 | if (version != 9) { |
181 | 0 | throw std::runtime_error("Unsupported BDB data file version number"); |
182 | 0 | } |
183 | | |
184 | | // Page size must be 512 <= pagesize <= 64k, and be a power of 2 |
185 | 0 | if (pagesize < 512 || pagesize > 65536 || (pagesize & (pagesize - 1)) != 0) { |
186 | 0 | throw std::runtime_error("Bad page size"); |
187 | 0 | } |
188 | | |
189 | | // Page type must be the btree type |
190 | 0 | if (type != PageType::BTREE_META) { |
191 | 0 | throw std::runtime_error("Unexpected page type, should be 9 (BTree Metadata)"); |
192 | 0 | } |
193 | | |
194 | | // Only supported meta-flag is subdatabase |
195 | 0 | if (flags != BTreeFlags::SUBDB) { |
196 | 0 | throw std::runtime_error("Unexpected database flags, should only be 0x20 (subdatabases)"); |
197 | 0 | } |
198 | 0 | } |
199 | | }; |
200 | | |
201 | | /** General class for records in a BDB BTree database. Contains common fields. */ |
202 | | class RecordHeader |
203 | | { |
204 | | public: |
205 | | uint16_t len; // Key/data item length |
206 | | RecordType type; // Page type (BDB has this include a DELETE FLAG that we track separately) |
207 | | bool deleted; // Whether the DELETE flag was set on type |
208 | | |
209 | | static constexpr size_t SIZE = 3; // The record header is 3 bytes |
210 | | |
211 | | bool other_endian; |
212 | | |
213 | 0 | RecordHeader(bool other_endian) : other_endian(other_endian) {} |
214 | | RecordHeader() = delete; |
215 | | |
216 | | template <typename Stream> |
217 | | void Unserialize(Stream& s) |
218 | 0 | { |
219 | 0 | s >> len; |
220 | |
|
221 | 0 | uint8_t uint8_type; |
222 | 0 | s >> uint8_type; |
223 | 0 | type = static_cast<RecordType>(uint8_type & ~static_cast<uint8_t>(RecordType::DELETE)); |
224 | 0 | deleted = uint8_type & static_cast<uint8_t>(RecordType::DELETE); |
225 | |
|
226 | 0 | if (other_endian) { |
227 | 0 | len = internal_bswap_16(len); |
228 | 0 | } |
229 | 0 | } |
230 | | }; |
231 | | |
232 | | /** Class for data in the record directly */ |
233 | | class DataRecord |
234 | | { |
235 | | public: |
236 | 0 | DataRecord(const RecordHeader& header) : m_header(header) {} |
237 | | DataRecord() = delete; |
238 | | |
239 | | RecordHeader m_header; |
240 | | |
241 | | std::vector<std::byte> data; // Variable length key/data item |
242 | | |
243 | | template <typename Stream> |
244 | | void Unserialize(Stream& s) |
245 | 0 | { |
246 | 0 | data.resize(m_header.len); |
247 | 0 | s.read(AsWritableBytes(Span(data.data(), data.size()))); |
248 | 0 | } |
249 | | }; |
250 | | |
251 | | /** Class for records representing internal nodes of the BTree. */ |
252 | | class InternalRecord |
253 | | { |
254 | | public: |
255 | 0 | InternalRecord(const RecordHeader& header) : m_header(header) {} |
256 | | InternalRecord() = delete; |
257 | | |
258 | | RecordHeader m_header; |
259 | | |
260 | | uint8_t unused; // Padding, unused |
261 | | uint32_t page_num; // Page number of referenced page |
262 | | uint32_t records; // Subtree record count |
263 | | std::vector<std::byte> data; // Variable length key item |
264 | | |
265 | | static constexpr size_t FIXED_SIZE = 9; // Size of fixed data is 9 bytes |
266 | | |
267 | | template <typename Stream> |
268 | | void Unserialize(Stream& s) |
269 | 0 | { |
270 | 0 | s >> unused; |
271 | 0 | s >> page_num; |
272 | 0 | s >> records; |
273 | |
|
274 | 0 | data.resize(m_header.len); |
275 | 0 | s.read(AsWritableBytes(Span(data.data(), data.size()))); |
276 | |
|
277 | 0 | if (m_header.other_endian) { |
278 | 0 | page_num = internal_bswap_32(page_num); |
279 | 0 | records = internal_bswap_32(records); |
280 | 0 | } |
281 | 0 | } |
282 | | }; |
283 | | |
284 | | /** Class for records representing overflow records of the BTree. |
285 | | * Overflow records point to a page which contains the data in the record. |
286 | | * Those pages may point to further pages with the rest of the data if it does not fit |
287 | | * in one page */ |
288 | | class OverflowRecord |
289 | | { |
290 | | public: |
291 | 0 | OverflowRecord(const RecordHeader& header) : m_header(header) {} |
292 | | OverflowRecord() = delete; |
293 | | |
294 | | RecordHeader m_header; |
295 | | |
296 | | uint8_t unused2; // Padding, unused |
297 | | uint32_t page_number; // Page number where data begins |
298 | | uint32_t item_len; // Total length of item |
299 | | |
300 | | static constexpr size_t SIZE = 9; // Overflow record is always 9 bytes |
301 | | |
302 | | template <typename Stream> |
303 | | void Unserialize(Stream& s) |
304 | 0 | { |
305 | 0 | s >> unused2; |
306 | 0 | s >> page_number; |
307 | 0 | s >> item_len; |
308 | |
|
309 | 0 | if (m_header.other_endian) { |
310 | 0 | page_number = internal_bswap_32(page_number); |
311 | 0 | item_len = internal_bswap_32(item_len); |
312 | 0 | } |
313 | 0 | } |
314 | | }; |
315 | | |
316 | | /** A generic data page in the database. Contains fields common to all data pages. */ |
317 | | class PageHeader |
318 | | { |
319 | | public: |
320 | | uint32_t lsn_file; // Log Sequence Number file |
321 | | uint32_t lsn_offset; // Log Sequence Number offset |
322 | | uint32_t page_num; // Current page number |
323 | | uint32_t prev_page; // Previous page number |
324 | | uint32_t next_page; // Next page number |
325 | | uint16_t entries; // Number of items on the page |
326 | | uint16_t hf_offset; // High free byte page offset |
327 | | uint8_t level; // Btree page level |
328 | | PageType type; // Page type |
329 | | |
330 | | static constexpr int64_t SIZE = 26; // The header is 26 bytes |
331 | | |
332 | | uint32_t expected_page_num; |
333 | | bool other_endian; |
334 | | |
335 | 0 | PageHeader(uint32_t page_num, bool other_endian) : expected_page_num(page_num), other_endian(other_endian) {} |
336 | | PageHeader() = delete; |
337 | | |
338 | | template <typename Stream> |
339 | | void Unserialize(Stream& s) |
340 | 0 | { |
341 | 0 | s >> lsn_file; |
342 | 0 | s >> lsn_offset; |
343 | 0 | s >> page_num; |
344 | 0 | s >> prev_page; |
345 | 0 | s >> next_page; |
346 | 0 | s >> entries; |
347 | 0 | s >> hf_offset; |
348 | 0 | s >> level; |
349 | |
|
350 | 0 | uint8_t uint8_type; |
351 | 0 | s >> uint8_type; |
352 | 0 | type = static_cast<PageType>(uint8_type); |
353 | |
|
354 | 0 | if (other_endian) { |
355 | 0 | lsn_file = internal_bswap_32(lsn_file); |
356 | 0 | lsn_offset = internal_bswap_32(lsn_offset); |
357 | 0 | page_num = internal_bswap_32(page_num); |
358 | 0 | prev_page = internal_bswap_32(prev_page); |
359 | 0 | next_page = internal_bswap_32(next_page); |
360 | 0 | entries = internal_bswap_16(entries); |
361 | 0 | hf_offset = internal_bswap_16(hf_offset); |
362 | 0 | } |
363 | |
|
364 | 0 | if (expected_page_num != page_num) { |
365 | 0 | throw std::runtime_error("Page number mismatch"); |
366 | 0 | } |
367 | 0 | if ((type != PageType::OVERFLOW_DATA && level < 1) || (type == PageType::OVERFLOW_DATA && level != 0)) { |
368 | 0 | throw std::runtime_error("Bad btree level"); |
369 | 0 | } |
370 | 0 | } |
371 | | }; |
372 | | |
373 | | /** A page of records in the database */ |
374 | | class RecordsPage |
375 | | { |
376 | | public: |
377 | 0 | RecordsPage(const PageHeader& header) : m_header(header) {} |
378 | | RecordsPage() = delete; |
379 | | |
380 | | PageHeader m_header; |
381 | | |
382 | | std::vector<uint16_t> indexes; |
383 | | std::vector<std::variant<DataRecord, OverflowRecord>> records; |
384 | | |
385 | | template <typename Stream> |
386 | | void Unserialize(Stream& s) |
387 | 0 | { |
388 | | // Current position within the page |
389 | 0 | int64_t pos = PageHeader::SIZE; |
390 | | |
391 | | // Get the items |
392 | 0 | for (uint32_t i = 0; i < m_header.entries; ++i) { |
393 | | // Get the index |
394 | 0 | uint16_t index; |
395 | 0 | s >> index; |
396 | 0 | if (m_header.other_endian) { |
397 | 0 | index = internal_bswap_16(index); |
398 | 0 | } |
399 | 0 | indexes.push_back(index); |
400 | 0 | pos += sizeof(uint16_t); |
401 | | |
402 | | // Go to the offset from the index |
403 | 0 | int64_t to_jump = index - pos; |
404 | 0 | if (to_jump < 0) { |
405 | 0 | throw std::runtime_error("Data record position not in page"); |
406 | 0 | } |
407 | 0 | s.ignore(to_jump); |
408 | | |
409 | | // Read the record |
410 | 0 | RecordHeader rec_hdr(m_header.other_endian); |
411 | 0 | s >> rec_hdr; |
412 | 0 | to_jump += RecordHeader::SIZE; |
413 | |
|
414 | 0 | switch (rec_hdr.type) { |
415 | 0 | case RecordType::KEYDATA: { |
416 | 0 | DataRecord record(rec_hdr); |
417 | 0 | s >> record; |
418 | 0 | records.emplace_back(record); |
419 | 0 | to_jump += rec_hdr.len; |
420 | 0 | break; |
421 | 0 | } |
422 | 0 | case RecordType::OVERFLOW_DATA: { |
423 | 0 | OverflowRecord record(rec_hdr); |
424 | 0 | s >> record; |
425 | 0 | records.emplace_back(record); |
426 | 0 | to_jump += OverflowRecord::SIZE; |
427 | 0 | break; |
428 | 0 | } |
429 | 0 | default: |
430 | 0 | throw std::runtime_error("Unknown record type in records page"); |
431 | 0 | } |
432 | | |
433 | | // Go back to the indexes |
434 | 0 | s.seek(-to_jump, SEEK_CUR); |
435 | 0 | } |
436 | 0 | } |
437 | | }; |
438 | | |
439 | | /** A page containing overflow data */ |
440 | | class OverflowPage |
441 | | { |
442 | | public: |
443 | 0 | OverflowPage(const PageHeader& header) : m_header(header) {} |
444 | | OverflowPage() = delete; |
445 | | |
446 | | PageHeader m_header; |
447 | | |
448 | | // BDB overloads some page fields to store overflow page data |
449 | | // hf_offset contains the length of the overflow data stored on this page |
450 | | // entries contains a reference count for references to this item |
451 | | |
452 | | // The overflow data itself. Begins immediately following header |
453 | | std::vector<std::byte> data; |
454 | | |
455 | | template <typename Stream> |
456 | | void Unserialize(Stream& s) |
457 | 0 | { |
458 | 0 | data.resize(m_header.hf_offset); |
459 | 0 | s.read(AsWritableBytes(Span(data.data(), data.size()))); |
460 | 0 | } |
461 | | }; |
462 | | |
463 | | /** A page of records in the database */ |
464 | | class InternalPage |
465 | | { |
466 | | public: |
467 | 0 | InternalPage(const PageHeader& header) : m_header(header) {} |
468 | | InternalPage() = delete; |
469 | | |
470 | | PageHeader m_header; |
471 | | |
472 | | std::vector<uint16_t> indexes; |
473 | | std::vector<InternalRecord> records; |
474 | | |
475 | | template <typename Stream> |
476 | | void Unserialize(Stream& s) |
477 | 0 | { |
478 | | // Current position within the page |
479 | 0 | int64_t pos = PageHeader::SIZE; |
480 | | |
481 | | // Get the items |
482 | 0 | for (uint32_t i = 0; i < m_header.entries; ++i) { |
483 | | // Get the index |
484 | 0 | uint16_t index; |
485 | 0 | s >> index; |
486 | 0 | if (m_header.other_endian) { |
487 | 0 | index = internal_bswap_16(index); |
488 | 0 | } |
489 | 0 | indexes.push_back(index); |
490 | 0 | pos += sizeof(uint16_t); |
491 | | |
492 | | // Go to the offset from the index |
493 | 0 | int64_t to_jump = index - pos; |
494 | 0 | if (to_jump < 0) { |
495 | 0 | throw std::runtime_error("Internal record position not in page"); |
496 | 0 | } |
497 | 0 | s.ignore(to_jump); |
498 | | |
499 | | // Read the record |
500 | 0 | RecordHeader rec_hdr(m_header.other_endian); |
501 | 0 | s >> rec_hdr; |
502 | 0 | to_jump += RecordHeader::SIZE; |
503 | |
|
504 | 0 | if (rec_hdr.type != RecordType::KEYDATA) { |
505 | 0 | throw std::runtime_error("Unknown record type in internal page"); |
506 | 0 | } |
507 | 0 | InternalRecord record(rec_hdr); |
508 | 0 | s >> record; |
509 | 0 | records.emplace_back(record); |
510 | 0 | to_jump += InternalRecord::FIXED_SIZE + rec_hdr.len; |
511 | | |
512 | | // Go back to the indexes |
513 | 0 | s.seek(-to_jump, SEEK_CUR); |
514 | 0 | } |
515 | 0 | } |
516 | | }; |
517 | | |
518 | | static void SeekToPage(AutoFile& s, uint32_t page_num, uint32_t page_size) |
519 | 0 | { |
520 | 0 | int64_t pos = int64_t{page_num} * page_size; |
521 | 0 | s.seek(pos, SEEK_SET); |
522 | 0 | } |
523 | | |
524 | | void BerkeleyRODatabase::Open() |
525 | 0 | { |
526 | | // Open the file |
527 | 0 | FILE* file = fsbridge::fopen(m_filepath, "rb"); |
528 | 0 | AutoFile db_file(file); |
529 | 0 | if (db_file.IsNull()) { |
530 | 0 | throw std::runtime_error("BerkeleyRODatabase: Failed to open database file"); |
531 | 0 | } |
532 | | |
533 | 0 | uint32_t page_size = 4096; // Default page size |
534 | | |
535 | | // Read the outer metapage |
536 | | // Expected page number is 0 |
537 | 0 | MetaPage outer_meta(0); |
538 | 0 | db_file >> outer_meta; |
539 | 0 | page_size = outer_meta.pagesize; |
540 | | |
541 | | // Verify the size of the file is a multiple of the page size |
542 | 0 | db_file.seek(0, SEEK_END); |
543 | 0 | int64_t size = db_file.tell(); |
544 | | |
545 | | // Since BDB stores everything in a page, the file size should be a multiple of the page size; |
546 | | // However, BDB doesn't actually check that this is the case, and enforcing this check results |
547 | | // in us rejecting a database that BDB would not, so this check needs to be excluded. |
548 | | // This is left commented out as a reminder to not accidentally implement this in the future. |
549 | | // if (size % page_size != 0) { |
550 | | // throw std::runtime_error("File size is not a multiple of page size"); |
551 | | // } |
552 | | |
553 | | // Check the last page number |
554 | 0 | uint32_t expected_last_page{uint32_t((size / page_size) - 1)}; |
555 | 0 | if (outer_meta.last_page != expected_last_page) { |
556 | 0 | throw std::runtime_error("Last page number could not fit in file"); |
557 | 0 | } |
558 | | |
559 | | // Make sure encryption is disabled |
560 | 0 | if (outer_meta.encrypt_algo != 0) { |
561 | 0 | throw std::runtime_error("BDB builtin encryption is not supported"); |
562 | 0 | } |
563 | | |
564 | | // Check all Log Sequence Numbers (LSN) point to file 0 and offset 1 which indicates that the LSNs were |
565 | | // reset and that the log files are not necessary to get all of the data in the database. |
566 | 0 | for (uint32_t i = 0; i < outer_meta.last_page; ++i) { |
567 | | // The LSN is composed of 2 32-bit ints, the first is a file id, the second an offset |
568 | | // It will always be the first 8 bytes of a page, so we deserialize it directly for every page |
569 | 0 | uint32_t file; |
570 | 0 | uint32_t offset; |
571 | 0 | SeekToPage(db_file, i, page_size); |
572 | 0 | db_file >> file >> offset; |
573 | 0 | if (outer_meta.other_endian) { |
574 | 0 | file = internal_bswap_32(file); |
575 | 0 | offset = internal_bswap_32(offset); |
576 | 0 | } |
577 | 0 | if (file != 0 || offset != 1) { |
578 | 0 | throw std::runtime_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support"); |
579 | 0 | } |
580 | 0 | } |
581 | | |
582 | | // Read the root page |
583 | 0 | SeekToPage(db_file, outer_meta.root, page_size); |
584 | 0 | PageHeader header(outer_meta.root, outer_meta.other_endian); |
585 | 0 | db_file >> header; |
586 | 0 | if (header.type != PageType::BTREE_LEAF) { |
587 | 0 | throw std::runtime_error("Unexpected outer database root page type"); |
588 | 0 | } |
589 | 0 | if (header.entries != 2) { |
590 | 0 | throw std::runtime_error("Unexpected number of entries in outer database root page"); |
591 | 0 | } |
592 | 0 | RecordsPage page(header); |
593 | 0 | db_file >> page; |
594 | | |
595 | | // First record should be the string "main" |
596 | 0 | if (!std::holds_alternative<DataRecord>(page.records.at(0)) || std::get<DataRecord>(page.records.at(0)).data != SUBDATABASE_NAME) { |
597 | 0 | throw std::runtime_error("Subdatabase has an unexpected name"); |
598 | 0 | } |
599 | | // Check length of page number for subdatabase location |
600 | 0 | if (!std::holds_alternative<DataRecord>(page.records.at(1)) || std::get<DataRecord>(page.records.at(1)).m_header.len != 4) { |
601 | 0 | throw std::runtime_error("Subdatabase page number has unexpected length"); |
602 | 0 | } |
603 | | |
604 | | // Read subdatabase page number |
605 | | // It is written as a big endian 32 bit number |
606 | 0 | uint32_t main_db_page = ReadBE32(UCharCast(std::get<DataRecord>(page.records.at(1)).data.data())); |
607 | | |
608 | | // The main database is in a page that doesn't exist |
609 | 0 | if (main_db_page > outer_meta.last_page) { |
610 | 0 | throw std::runtime_error("Page number is greater than database last page"); |
611 | 0 | } |
612 | | |
613 | | // Read the inner metapage |
614 | 0 | SeekToPage(db_file, main_db_page, page_size); |
615 | 0 | MetaPage inner_meta(main_db_page); |
616 | 0 | db_file >> inner_meta; |
617 | |
|
618 | 0 | if (inner_meta.pagesize != page_size) { |
619 | 0 | throw std::runtime_error("Unexpected page size"); |
620 | 0 | } |
621 | | |
622 | 0 | if (inner_meta.last_page > outer_meta.last_page) { |
623 | 0 | throw std::runtime_error("Subdatabase last page is greater than database last page"); |
624 | 0 | } |
625 | | |
626 | | // Make sure encryption is disabled |
627 | 0 | if (inner_meta.encrypt_algo != 0) { |
628 | 0 | throw std::runtime_error("BDB builtin encryption is not supported"); |
629 | 0 | } |
630 | | |
631 | | // Do a DFS through the BTree, starting at root |
632 | 0 | std::vector<uint32_t> pages{inner_meta.root}; |
633 | 0 | while (pages.size() > 0) { |
634 | 0 | uint32_t curr_page = pages.back(); |
635 | | // It turns out BDB completely ignores this last_page field and doesn't actually update it to the correct |
636 | | // last page. While we should be checking this, we can't. |
637 | | // This is left commented out as a reminder to not accidentally implement this in the future. |
638 | | // if (curr_page > inner_meta.last_page) { |
639 | | // throw std::runtime_error("Page number is greater than subdatabase last page"); |
640 | | // } |
641 | 0 | pages.pop_back(); |
642 | 0 | SeekToPage(db_file, curr_page, page_size); |
643 | 0 | PageHeader header(curr_page, inner_meta.other_endian); |
644 | 0 | db_file >> header; |
645 | 0 | switch (header.type) { |
646 | 0 | case PageType::BTREE_INTERNAL: { |
647 | 0 | InternalPage int_page(header); |
648 | 0 | db_file >> int_page; |
649 | 0 | for (const InternalRecord& rec : int_page.records) { |
650 | 0 | if (rec.m_header.deleted) continue; |
651 | 0 | pages.push_back(rec.page_num); |
652 | 0 | } |
653 | 0 | break; |
654 | 0 | } |
655 | 0 | case PageType::BTREE_LEAF: { |
656 | 0 | RecordsPage rec_page(header); |
657 | 0 | db_file >> rec_page; |
658 | 0 | if (rec_page.records.size() % 2 != 0) { |
659 | | // BDB stores key value pairs in consecutive records, thus an odd number of records is unexpected |
660 | 0 | throw std::runtime_error("Records page has odd number of records"); |
661 | 0 | } |
662 | 0 | bool is_key = true; |
663 | 0 | std::vector<std::byte> key; |
664 | 0 | for (const std::variant<DataRecord, OverflowRecord>& rec : rec_page.records) { |
665 | 0 | std::vector<std::byte> data; |
666 | 0 | if (const DataRecord* drec = std::get_if<DataRecord>(&rec)) { |
667 | 0 | if (drec->m_header.deleted) continue; |
668 | 0 | data = drec->data; |
669 | 0 | } else if (const OverflowRecord* orec = std::get_if<OverflowRecord>(&rec)) { |
670 | 0 | if (orec->m_header.deleted) continue; |
671 | 0 | uint32_t next_page = orec->page_number; |
672 | 0 | while (next_page != 0) { |
673 | 0 | SeekToPage(db_file, next_page, page_size); |
674 | 0 | PageHeader opage_header(next_page, inner_meta.other_endian); |
675 | 0 | db_file >> opage_header; |
676 | 0 | if (opage_header.type != PageType::OVERFLOW_DATA) { |
677 | 0 | throw std::runtime_error("Bad overflow record page type"); |
678 | 0 | } |
679 | 0 | OverflowPage opage(opage_header); |
680 | 0 | db_file >> opage; |
681 | 0 | data.insert(data.end(), opage.data.begin(), opage.data.end()); |
682 | 0 | next_page = opage_header.next_page; |
683 | 0 | } |
684 | 0 | } |
685 | | |
686 | 0 | if (is_key) { |
687 | 0 | key = data; |
688 | 0 | } else { |
689 | 0 | m_records.emplace(SerializeData{key.begin(), key.end()}, SerializeData{data.begin(), data.end()}); |
690 | 0 | key.clear(); |
691 | 0 | } |
692 | 0 | is_key = !is_key; |
693 | 0 | } |
694 | 0 | break; |
695 | 0 | } |
696 | 0 | default: |
697 | 0 | throw std::runtime_error("Unexpected page type"); |
698 | 0 | } |
699 | 0 | } |
700 | 0 | } |
701 | | |
702 | | std::unique_ptr<DatabaseBatch> BerkeleyRODatabase::MakeBatch(bool flush_on_close) |
703 | 0 | { |
704 | 0 | return std::make_unique<BerkeleyROBatch>(*this); |
705 | 0 | } |
706 | | |
707 | | bool BerkeleyRODatabase::Backup(const std::string& dest) const |
708 | 0 | { |
709 | 0 | fs::path src(m_filepath); |
710 | 0 | fs::path dst(fs::PathFromString(dest)); |
711 | |
|
712 | 0 | if (fs::is_directory(dst)) { |
713 | 0 | dst = BDBDataFile(dst); |
714 | 0 | } |
715 | 0 | try { |
716 | 0 | if (fs::exists(dst) && fs::equivalent(src, dst)) { |
717 | 0 | LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(dst)); |
718 | 0 | return false; |
719 | 0 | } |
720 | | |
721 | 0 | fs::copy_file(src, dst, fs::copy_options::overwrite_existing); |
722 | 0 | LogPrintf("copied %s to %s\n", fs::PathToString(m_filepath), fs::PathToString(dst)); |
723 | 0 | return true; |
724 | 0 | } catch (const fs::filesystem_error& e) { |
725 | 0 | LogPrintf("error copying %s to %s - %s\n", fs::PathToString(m_filepath), fs::PathToString(dst), fsbridge::get_filesystem_error_message(e)); |
726 | 0 | return false; |
727 | 0 | } |
728 | 0 | } |
729 | | |
730 | | bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value) |
731 | 0 | { |
732 | 0 | SerializeData key_data{key.begin(), key.end()}; |
733 | 0 | const auto it{m_database.m_records.find(key_data)}; |
734 | 0 | if (it == m_database.m_records.end()) { |
735 | 0 | return false; |
736 | 0 | } |
737 | 0 | auto val = it->second; |
738 | 0 | value.clear(); |
739 | 0 | value.write(Span(val)); |
740 | 0 | return true; |
741 | 0 | } |
742 | | |
743 | | bool BerkeleyROBatch::HasKey(DataStream&& key) |
744 | 0 | { |
745 | 0 | SerializeData key_data{key.begin(), key.end()}; |
746 | 0 | return m_database.m_records.count(key_data) > 0; |
747 | 0 | } |
748 | | |
749 | | BerkeleyROCursor::BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix) |
750 | 0 | : m_database(database) |
751 | 0 | { |
752 | 0 | std::tie(m_cursor, m_cursor_end) = m_database.m_records.equal_range(BytePrefix{prefix}); |
753 | 0 | } |
754 | | |
755 | | DatabaseCursor::Status BerkeleyROCursor::Next(DataStream& ssKey, DataStream& ssValue) |
756 | 0 | { |
757 | 0 | if (m_cursor == m_cursor_end) { |
758 | 0 | return DatabaseCursor::Status::DONE; |
759 | 0 | } |
760 | 0 | ssKey.write(Span(m_cursor->first)); |
761 | 0 | ssValue.write(Span(m_cursor->second)); |
762 | 0 | m_cursor++; |
763 | 0 | return DatabaseCursor::Status::MORE; |
764 | 0 | } |
765 | | |
766 | | std::unique_ptr<DatabaseCursor> BerkeleyROBatch::GetNewPrefixCursor(Span<const std::byte> prefix) |
767 | 0 | { |
768 | 0 | return std::make_unique<BerkeleyROCursor>(m_database, prefix); |
769 | 0 | } |
770 | | |
771 | | std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) |
772 | 0 | { |
773 | 0 | fs::path data_file = BDBDataFile(path); |
774 | 0 | try { |
775 | 0 | std::unique_ptr<BerkeleyRODatabase> db = std::make_unique<BerkeleyRODatabase>(data_file); |
776 | 0 | status = DatabaseStatus::SUCCESS; |
777 | 0 | return db; |
778 | 0 | } catch (const std::runtime_error& e) { |
779 | 0 | error.original = e.what(); |
780 | 0 | status = DatabaseStatus::FAILED_LOAD; |
781 | 0 | return nullptr; |
782 | 0 | } |
783 | 0 | } |
784 | | } // namespace wallet |