Repository navigation
Expand file tree
/
Copy pathtest_index.cpp
More file actions
6121 lines (5178 loc) · 277 KB
/
Copy pathtest_index.cpp
File metadata and controls
6121 lines (5178 loc) · 277 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// This file is part of the Diffractor photo and video organizer
// Copyright 2026 Zac Walker
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
// Purpose: Index and database tests. Verifies the inverted/trigram/postings index, folder scanning
// and roots parsing, the SQLite cache schema and storage, thumbnail publication, staging and
// trimming, item property and metadata caching, duplicate and presence reporting, and cloud
// placeholder hydration.
#include "pch.h"
#include <condition_variable>
#include <sqlite3.h>
#include "test_fixtures.h"
#include "model_db_pack.h"
#include "model_postings.h"
#include "util_crash_files_db.h"
#include "app_util.h"
#include "metadata.h"
#include "ui_elements.h"
#include "ui_map_common.h"
extern std::function<void(core::folder_path folder, int attempt)> test_after_validate_folder_snapshot;
extern std::function<void(const df::index_file_item& anchor)> test_after_duplicate_anchor;
static void should_create_database_schema()
{
sqlite3_initialize();
const auto database_path = test_temps.next_path(".db");
sqlite3* database_handle = nullptr;
const auto open_result = sqlite3_open(database_path.str().c_str(), &database_handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw_db(database_handle, sqlite3_close);
if (open_result != SQLITE_OK)
{
throw test_assert_exception(std::format("Failed to create schema test database: {}",
raw_db ? sqlite3_errmsg(raw_db.get()) : "out of memory"));
}
const auto resource = load_app_resource(app_resource::sql);
const std::string schema(reinterpret_cast<const char*>(resource.data()), resource.size());
if (sqlite3_exec(raw_db.get(), schema.c_str(), nullptr, nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception(std::format("Failed to execute database schema: {}", sqlite3_errmsg(raw_db.get())));
}
const auto query_text = [&raw_db](const std::string_view sql)
{
sqlite3_stmt* statement_handle = nullptr;
const auto prepare_result = sqlite3_prepare_v2(raw_db.get(), sql.data(), static_cast<int>(sql.size()),
&statement_handle, nullptr);
const std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> statement(statement_handle, sqlite3_finalize);
if (prepare_result != SQLITE_OK || sqlite3_step(statement.get()) != SQLITE_ROW)
{
throw test_assert_exception(std::format("Failed schema query '{}': {}", sql, sqlite3_errmsg(raw_db.get())));
}
const auto* text = sqlite3_column_text(statement.get(), 0);
return text == nullptr ? std::string{} : std::string(reinterpret_cast<const char*>(text));
};
const auto raw_journal_mode = query_text("PRAGMA journal_mode");
assert_equal(true, !raw_journal_mode.empty(), "schema leaves journal mode to the database owner");
assert_equal("1", query_text("PRAGMA synchronous"), "schema synchronous mode");
assert_equal("item_imports,item_properties,item_thumbnails,web_service_cache",
query_text("SELECT group_concat(name, ',') FROM (SELECT name FROM sqlite_schema "
"WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name)"),
"schema tables");
assert_equal("folder,name,properties,hash,media_position,flag,crc,last_scanned,last_indexed",
query_text("SELECT group_concat(name, ',') FROM pragma_table_info('item_properties')"),
"item_properties columns");
assert_equal("folder,name,bitmap,cover_art,last_scanned",
query_text("SELECT group_concat(name, ',') FROM pragma_table_info('item_thumbnails')"),
"item_thumbnails columns");
assert_equal("key,created_date,value",
query_text("SELECT group_concat(name, ',') FROM pragma_table_info('web_service_cache')"),
"web_service_cache columns");
assert_equal("name,modified,size,imported",
query_text("SELECT group_concat(name, ',') FROM pragma_table_info('item_imports')"),
"item_imports columns");
assert_equal("folder,name",
query_text("SELECT group_concat(name, ',') FROM "
"(SELECT name FROM pragma_table_info('item_properties') WHERE pk > 0 ORDER BY pk)"),
"item_properties primary key");
assert_equal("folder,name",
query_text("SELECT group_concat(name, ',') FROM "
"(SELECT name FROM pragma_table_info('item_thumbnails') WHERE pk > 0 ORDER BY pk)"),
"item_thumbnails primary key");
assert_equal("key",
query_text("SELECT group_concat(name, ',') FROM "
"(SELECT name FROM pragma_table_info('web_service_cache') WHERE pk > 0 ORDER BY pk)"),
"web_service_cache primary key");
assert_equal("name,modified,size",
query_text("SELECT group_concat(name, ',') FROM "
"(SELECT name FROM pragma_table_info('item_imports') WHERE pk > 0 ORDER BY pk)"),
"item_imports primary key");
assert_equal("ok", query_text("PRAGMA integrity_check"), "schema integrity");
}
static void should_open_database_in_wal_mode()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
{
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
db.close();
}
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (open_result != SQLITE_OK) throw test_assert_exception("Failed to open database journal test"s);
sqlite3_stmt* stmt = nullptr;
if (sqlite3_prepare_v2(raw.get(), "PRAGMA journal_mode", -1, &stmt, nullptr) != SQLITE_OK)
throw test_assert_exception("Failed to read journal mode"s);
const std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> statement(stmt, sqlite3_finalize);
assert_equal(SQLITE_ROW, sqlite3_step(statement.get()), "journal mode row");
const auto* text = sqlite3_column_text(statement.get(), 0);
assert_equal("wal", std::string(reinterpret_cast<const char*>(text)), "database owner selects WAL");
}
static void should_keep_rollback_journal_schema_initialization()
{
sqlite3_initialize();
const auto database_path = test_temps.next_path(".db");
sqlite3* database_handle = nullptr;
const auto open_result = sqlite3_open(database_path.str().c_str(), &database_handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw_db(database_handle, sqlite3_close);
if (open_result != SQLITE_OK) throw test_assert_exception("Failed to open rollback journal test database"s);
assert_equal(SQLITE_OK, database_test_seams::set_journal_mode(raw_db.get(), "DELETE"sv),
"rollback journal can be selected");
const auto resource = load_app_resource(app_resource::sql);
const std::string schema(reinterpret_cast<const char*>(resource.data()), resource.size());
if (sqlite3_exec(raw_db.get(), schema.c_str(), nullptr, nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception("Failed to execute schema after rollback selection"s);
}
sqlite3_stmt* stmt = nullptr;
if (sqlite3_prepare_v2(raw_db.get(), "PRAGMA journal_mode", -1, &stmt, nullptr) != SQLITE_OK)
throw test_assert_exception("Failed to read retained journal mode"s);
const std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> statement(stmt, sqlite3_finalize);
assert_equal(SQLITE_ROW, sqlite3_step(statement.get()), "journal mode row");
const auto* text = sqlite3_column_text(statement.get(), 0);
const auto retained_mode = std::string(reinterpret_cast<const char*>(text));
assert_equal(true, !retained_mode.empty(), "schema creation keeps a journal mode selected by SQLite");
sqlite3* memory_handle = nullptr;
if (sqlite3_open(":memory:", &memory_handle) != SQLITE_OK)
throw test_assert_exception("Failed to open memory database"s);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> memory(memory_handle, sqlite3_close);
assert_equal(true, database_test_seams::set_journal_mode(memory.get(), "WAL"sv) != SQLITE_OK,
"failure to establish the requested mode is reported");
const auto assert_fallback_open = [](const core::file_path index_path, const std::string_view label)
{
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
database_test_seams::fail_next_journal_probe(SQLITE_IOERR_SHMOPEN);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
assert_equal(true, db.is_open(), std::format("{} opens after shared-memory failure", label));
db.close();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
sqlite3* handle = nullptr;
if (sqlite3_open(db_path.str().c_str(), &handle) != SQLITE_OK)
throw test_assert_exception("Failed to inspect fallback database"s);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
{
sqlite3_stmt* mode_stmt = nullptr;
if (sqlite3_prepare_v2(raw.get(), "PRAGMA journal_mode", -1, &mode_stmt, nullptr) != SQLITE_OK)
throw test_assert_exception("Failed to inspect fallback journal mode"s);
const std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> mode(mode_stmt, sqlite3_finalize);
assert_equal(SQLITE_ROW, sqlite3_step(mode.get()), "journal mode row");
const auto* text = sqlite3_column_text(mode.get(), 0);
assert_equal("delete", std::string(reinterpret_cast<const char*>(text)),
std::format("{} stays in rollback journal mode", label));
}
if (sqlite3_exec(raw.get(),
"BEGIN; INSERT OR REPLACE INTO web_service_cache VALUES ('probe', 1, 'ok'); COMMIT;", nullptr,
nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception(
std::format("{} could not write after fallback: {}", label, sqlite3_errmsg(raw.get())));
}
};
assert_fallback_open(test_temps.next_path(), "fresh database"sv);
const auto rollback_path = test_temps.next_path();
{
const auto rollback_db_path = core::file_path(rollback_path.folder(),
rollback_path.file_name_without_extension(), ".db");
sqlite3* handle = nullptr;
if (sqlite3_open(rollback_db_path.str().c_str(), &handle) != SQLITE_OK)
throw test_assert_exception("Failed to seed rollback database"s);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
assert_equal(SQLITE_OK, database_test_seams::set_journal_mode(raw.get(), "DELETE"sv), "seed rollback journal");
const auto seeded_schema = load_app_resource(app_resource::sql);
const std::string seeded_sql(reinterpret_cast<const char*>(seeded_schema.data()), seeded_schema.size());
if (sqlite3_exec(raw.get(), seeded_sql.c_str(), nullptr, nullptr, nullptr) != SQLITE_OK)
throw test_assert_exception("Failed to seed rollback schema"s);
}
assert_fallback_open(rollback_path, "rollback database"sv);
const auto wal_path = test_temps.next_path();
{
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
database db(index);
db.open(wal_path.folder(), wal_path.file_name_without_extension());
db.close();
}
assert_fallback_open(wal_path, "existing WAL database"sv);
}
static void should_classify_schema_check_failures_before_replacement()
{
assert_equal(false, database_test_seams::schema_failure_allows_replacement(SQLITE_BUSY),
"busy is environmental, not replaceable");
assert_equal(false, database_test_seams::schema_failure_allows_replacement(SQLITE_LOCKED),
"locked is environmental, not replaceable");
assert_equal(false, database_test_seams::schema_failure_allows_replacement(SQLITE_IOERR),
"I/O is environmental, not replaceable");
assert_equal(true, database_test_seams::schema_failure_allows_replacement(SQLITE_ERROR),
"incompatible schema is replaceable");
assert_equal(true, database_test_seams::schema_failure_allows_replacement(SQLITE_CORRUPT),
"corrupt bytes are replaceable");
assert_equal(true, database_test_seams::schema_failure_allows_replacement(SQLITE_NOTADB),
"non-database bytes are replaceable");
}
static void should_store_thumbnails()
{
const auto index_path = test_temps.next_path();
const auto file_path = test_files_folder().combine_file("Test.jpg");
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto i = load_item(index, file_path, true);
db.perform_writes();
const auto thumb = db.load_thumbnail(i->path());
assert_equal(i->thumbnail(), thumb.thumb, "local loaded thumb");
}
static void should_store_cover_art()
{
const auto index_path = test_temps.next_path();
const auto file_path = test_files_folder().combine_file("indy.mp4");
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto i = load_item(index, file_path, true);
db.perform_writes();
const auto thumb = db.load_thumbnail(i->path());
assert_equal(i->cover_art(), thumb.cover_art, "local loaded cover art");
}
static void should_store_item_properties()
{
const auto index_path = test_temps.next_path();
const auto file_path = test_files_folder().combine_file("Test.jpg");
null_async_strategy as;
location_cache locations;
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
auto data = platform::blob_from_file(file_path);
const auto crc32c_expected = core::crc32c(data.data(), data.size());
auto md = std::make_shared<prop::item_metadata>();
md->album = "test"_c;
md->orientation = core::orientation::bottom_right;
std::deque<item_db_write> writes;
constexpr auto media_pos = 111.1;
{
item_db_write w;
w.crc32c = crc32c_expected;
w.md = md;
w.media_position = media_pos;
w.path = file_path;
writes.emplace_back(std::move(w));
}
db.perform_writes(std::move(writes));
db.load_index_values();
auto item = index.find_item(file_path);
auto item_md = item.metadata.load();
assert_metadata(*md, *item_md, "index");
assert_equal(crc32c_expected, item.crc32c, "index crc32");
assert_equal(static_cast<int>(media_pos), static_cast<int>(item_md->media_position), "index media position");
assert_equal(md->orientation, item_md->orientation, "index orientation");
const auto reloaded_crc = platform::file_crc32(file_path);
assert_equal(reloaded_crc, item.crc32c, "platform::file_crc32 crc32");
}
// A rescan carries the position the index holds into the database row. A position saved only to the
// row was written back over by the next rescan of the file, so the next session resumed from wherever
// the index had been loaded.
static void should_round_trip_opaque_cache_properties()
{
assert_equal(true, prop::from_id(prop::cache::faces) == prop::null,
"the face cache is not a public media property");
const auto md = std::make_shared<prop::item_metadata>();
md->album = "public metadata"_c;
for (const auto size : {0_z, 253_z, 254_z, 65534_z, 65535_z, 65536_z})
{
core::blob payload(size);
for (auto i = 0_z; i < size; ++i) payload[i] = static_cast<uint8_t>(i * 17);
metadata_packer metadata;
metadata.pack(md);
metadata_packer packer;
packer.write_blob(prop::cache::faces, payload);
packer._data.insert(packer._data.end(), metadata._data.begin() + 2, metadata._data.end());
const auto found = metadata_find_blob_property(packer.cdata(), prop::cache::faces);
assert_equal(true, found.has_value(), "an empty value differs from an absent property");
assert_equal(size, found->size, "binary lengths round trip at both framing boundaries");
assert_equal(true, size == 0 || memcmp(found->data, payload.data(), size) == 0, "binary bytes round trip");
const auto loaded = std::make_shared<prop::item_metadata>();
metadata_unpacker reader(packer.cdata());
reader.unpack(loaded);
assert_equal(md->album, loaded->album, "unknown binary values do not hide following metadata");
const auto stripped = metadata_replace_blob_property(packer.cdata(), prop::cache::faces, std::nullopt);
assert_equal(false, metadata_find_blob_property(stripped, prop::cache::faces).has_value(),
"removal leaves no cache value");
metadata_packer expected;
expected.pack(md);
assert_equal(true, stripped == expected._data, "removal preserves every other property byte");
}
metadata_packer alias;
const auto original = alias._data.clone();
alias.write_blob(prop::cache::faces, alias.cdata());
const auto found = metadata_find_blob_property(alias.cdata(), prop::cache::faces);
assert_equal(true,
found && found->size == original.size() && memcmp(found->data, original.data(), original.size()) == 0,
"a borrowed value may refer to the packer's own buffer");
for (const auto& invalid : {core::blob{0xff}, core::blob{0xff, 1, 1}, core::blob{0xff, 1, 1, 0, 0xfe, 1},
core::blob{0xff, 1, 1, 0, 4, 0}})
{
auto rejected = false;
try
{
metadata_replace_blob_property(invalid, prop::cache::faces, std::nullopt);
}
catch (const metadata_property_error&)
{
rejected = true;
}
assert_equal(true, rejected, "malformed property storage is not rewritten as success");
}
}
static core::blob read_test_cached_properties(sqlite3* db, const core::file_path path)
{
sqlite3_stmt* handle = nullptr;
const auto prepared = sqlite3_prepare_v2(db, "select properties from item_properties where folder=? and name=?", -1,
&handle, nullptr);
const std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> statement(handle, sqlite3_finalize);
const auto folder = path.folder().text();
const auto name = std::string(path.name());
if (prepared != SQLITE_OK || sqlite3_bind_text(handle, 1, folder.sz(), -1, SQLITE_TRANSIENT) != SQLITE_OK ||
sqlite3_bind_text(handle, 2, name.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK ||
sqlite3_step(handle) != SQLITE_ROW)
throw test_assert_exception(std::format("Cannot read test cache: {}", sqlite3_errmsg(db)));
const auto size = sqlite3_column_bytes(handle, 0);
const auto* bytes = static_cast<const uint8_t*>(sqlite3_column_blob(handle, 0));
if (size && !bytes) throw test_assert_exception("Cannot read cached-property bytes");
return size ? core::blob(bytes, bytes + size) : core::blob{};
}
// A metadata-only rescan used to erase opaque cache records. Exercise the actual database writer.
static void should_preserve_opaque_properties_when_metadata_is_rewritten()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto path = test_files_folder().combine_file("Test.jpg");
null_async_strategy async;
const location_cache locations;
index_state index(async, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
auto md = std::make_shared<prop::item_metadata>();
md->album = "before"_c;
const auto save = [&db](item_db_write write)
{
std::deque<item_db_write> writes;
writes.emplace_back(std::move(write));
db.perform_writes(std::move(writes));
};
item_db_write initial;
initial.path = path;
initial.md = md;
initial.crc32c = 0x1234u;
initial.media_position = 17.0;
initial.phash = crypto::phash_rotations{11, 22, 33, 44};
save(std::move(initial));
sqlite3* handle = nullptr;
const auto opened = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (opened != SQLITE_OK) throw test_assert_exception("Cannot open test property cache");
const core::blob payload(65536, 0x5a);
const core::blob other{1, 2, 3};
auto stored = metadata_replace_blob_property(read_test_cached_properties(handle, path), prop::cache::faces,
core::cspan(payload));
stored = metadata_replace_blob_property(stored, 0xfffd, core::cspan(other));
{
sqlite3_stmt* update_handle = nullptr;
const auto prepared = sqlite3_prepare_v2(handle,
"update item_properties set properties=? where folder=? and name=?",
-1, &update_handle, nullptr);
const std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> update(update_handle, sqlite3_finalize);
const auto folder = path.folder().text();
const auto name = std::string(path.name());
if (prepared != SQLITE_OK ||
sqlite3_bind_blob(update_handle, 1, stored.data(), static_cast<int>(stored.size()), SQLITE_TRANSIENT) !=
SQLITE_OK ||
sqlite3_bind_text(update_handle, 2, folder.sz(), -1, SQLITE_TRANSIENT) != SQLITE_OK ||
sqlite3_bind_text(update_handle, 3, name.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK ||
sqlite3_step(update_handle) != SQLITE_DONE)
throw test_assert_exception("Cannot seed test cache properties");
}
auto changed = std::make_shared<prop::item_metadata>(*md);
changed->album = "after"_c;
item_db_write write;
write.path = path;
write.md = changed;
save(std::move(write));
const auto reloaded = read_test_cached_properties(handle, path);
const auto found = metadata_find_blob_property(reloaded, prop::cache::faces);
assert_equal(true,
found && found->size == payload.size() && memcmp(found->data, payload.data(), payload.size()) == 0,
"ordinary metadata writes preserve the opaque face-cache value");
const auto retained = metadata_find_blob_property(reloaded, 0xfffd);
assert_equal(true,
retained && retained->size == other.size() && memcmp(retained->data, other.data(), other.size()) == 0,
"other unknown properties survive too");
db.load_index_values();
const auto item = index.find_item(path);
const auto loaded_md = item.metadata.load();
assert_equal(changed->album, loaded_md->album, "known metadata still changes");
assert_equal(0x1234u, item.crc32c.load(), "CRC is retained");
assert_equal(17, static_cast<int>(loaded_md->media_position), "playback state is retained");
const auto hashes = item.phash.load();
assert_equal(true, hashes && hashes->rotations[0] == 11, "perceptual hashes are retained");
const auto empty_path = test_files_folder().combine_file("empty-cache.jpg");
item_db_write empty;
empty.path = empty_path;
empty.md = prop::item_metadata_ptr{};
save(std::move(empty));
const auto empty_properties = read_test_cached_properties(handle, empty_path);
assert_equal(false, metadata_find_blob_property(empty_properties, prop::cache::faces).has_value(),
"a cache value never leaks from the previous item");
}
static void should_hold_a_saved_playback_position_in_the_index()
{
const auto root = test_temps.next_folder("saved-playback-position");
const auto path = root.combine_file("played.jpg");
platform::copy_file(test_files_folder().combine_file("Test.jpg"), path, false, false);
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
core::index_roots roots;
roots.folders.emplace(root);
index.index_roots(roots);
index.index_folders(test_token);
index.scan_uncached(test_token);
assert_equal(true, index.find_item(path).metadata.load() != nullptr, "the file was scanned");
index.save_media_position(path, 42.0);
const auto md = index.find_item(path).metadata.load();
assert_equal(true, md != nullptr && static_cast<int>(md->media_position) == 42,
"the index holds the saved position, so a rescan carries it rather than the old one");
}
// A database written before db_metadata_version records place text that cannot be told apart from
// text the file itself carried, so opening it must drop the cached metadata and force a rescan.
// Everything a rescan does not replace is kept, or the upgrade would cost a full thumbnail rebuild.
static void should_invalidate_cached_metadata_written_by_an_older_build()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto file_path = test_files_folder().combine_file("Test.jpg");
null_async_strategy as;
location_cache locations;
{
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
auto md = std::make_shared<prop::item_metadata>();
md->album = "test"_c;
md->location_place = "Somewhere"_c;
std::deque<item_db_write> writes;
item_db_write w;
w.path = file_path;
w.md = md;
w.crc32c = 0x1234u;
w.media_position = 111.1;
w.metadata_scanned = core::date_t(2020, 1, 1, 0, 0, 0);
writes.emplace_back(std::move(w));
db.perform_writes(std::move(writes));
db.close();
}
// Stamp the file as one an older build left behind.
{
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (open_result != SQLITE_OK ||
sqlite3_exec(raw.get(), "PRAGMA user_version = 0;", nullptr, nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception("Failed to reset the database metadata version"s);
}
}
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto item = index.find_item(file_path);
const auto item_md = item.metadata.load();
assert_equal(0ll, item.metadata_scanned.load().to_int64(), "scan state cleared");
assert_equal(true, item_md == nullptr || core::is_empty(item_md->location_place), "stale place cleared");
assert_equal(0x1234u, item.crc32c, "crc retained");
assert_equal(true, item_md != nullptr && static_cast<int>(item_md->media_position) == 111,
"media position retained");
}
// Moving to the date pack asks for a re-read, but a pre-pack row still answers in the meantime. So
// this upgrade clears only the scan stamp and keeps the cached properties, or every search, group
// and timeline would go blank until the background re-index caught up.
static void should_keep_answering_while_upgrading_to_the_date_pack()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto file_path = test_files_folder().combine_file("Test.jpg");
null_async_strategy as;
location_cache locations;
{
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
auto md = std::make_shared<prop::item_metadata>();
md->album = "test"_c;
md->dates.add(prop::date_source::exif_original, core::date_t(2012, 9, 14, 19, 21, 14));
std::deque<item_db_write> writes;
item_db_write w;
w.path = file_path;
w.md = md;
w.crc32c = 0x1234u;
w.metadata_scanned = core::date_t(2020, 1, 1, 0, 0, 0);
writes.emplace_back(std::move(w));
db.perform_writes(std::move(writes));
db.close();
}
// A database written by the build before the pack, rather than one from before version 1.
{
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (open_result != SQLITE_OK ||
sqlite3_exec(raw.get(), "PRAGMA user_version = 1;", nullptr, nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception("Failed to set the database metadata version"s);
}
}
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto item = index.find_item(file_path);
const auto item_md = item.metadata.load();
assert_equal(0ll, item.metadata_scanned.load().to_int64(), "a re-scan is requested");
assert_equal(true, item_md != nullptr, "the cached properties are kept");
assert_equal(core::date_t(2012, 9, 14, 19, 21, 14), item_md->dates.original(),
"and still answer with the date they held");
assert_equal("test", item_md->album.sv(), "the rest of the cached metadata is untouched");
}
// Versions 3, 4 and 5 each changed only what an audio or video scan records, so each upgrade asks those
// items alone to be re-scanned, and every row keeps answering meanwhile.
static void should_keep_answering_while_upgrading_audio_metadata(const int stored_version)
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto photo_path = test_files_folder().combine_file("Test.jpg");
const auto audio_path = test_files_folder().combine_file("Colorblind.mp3");
null_async_strategy as;
location_cache locations;
{
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
auto photo_md = std::make_shared<prop::item_metadata>();
photo_md->album = "photo"_c;
auto audio_md = std::make_shared<prop::item_metadata>();
audio_md->album = "audio"_c;
audio_md->audio_sample_rate = 65535;
std::deque<item_db_write> writes;
item_db_write photo;
photo.path = photo_path;
photo.md = photo_md;
photo.metadata_scanned = core::date_t(2020, 1, 1, 0, 0, 0);
writes.emplace_back(std::move(photo));
item_db_write audio;
audio.path = audio_path;
audio.md = audio_md;
audio.metadata_scanned = core::date_t(2020, 1, 1, 0, 0, 0);
writes.emplace_back(std::move(audio));
db.perform_writes(std::move(writes));
db.close();
}
{
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (open_result != SQLITE_OK ||
sqlite3_exec(raw.get(), std::format("PRAGMA user_version = {};", stored_version).c_str(), nullptr, nullptr,
nullptr) != SQLITE_OK)
{
throw test_assert_exception("Failed to set the audio metadata version"s);
}
}
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto photo_item = index.find_item(photo_path);
const auto photo_md = photo_item.metadata.load();
const auto audio_item = index.find_item(audio_path);
const auto audio_md = audio_item.metadata.load();
assert_equal(core::date_t(2020, 1, 1, 0, 0, 0).to_int64(), photo_item.metadata_scanned.load().to_int64(),
"photo scan state is not cleared by the audio upgrade");
assert_equal(0ll, audio_item.metadata_scanned.load().to_int64(), "an audio re-scan is requested");
assert_equal(true, photo_md != nullptr && audio_md != nullptr, "cached metadata is kept");
assert_equal("photo", photo_md->album.sv(), "photo metadata still answers");
assert_equal("audio", audio_md->album.sv(), "audio metadata still answers");
assert_equal(65535u, audio_md->audio_sample_rate, "legacy saturated sample rate remains readable meanwhile");
}
// Version 6 recorded libopenmpt's stand-in for a missing song message - the module's sample and instrument
// names - as its comment. Only tracker modules are read again for it, and they keep answering meanwhile.
static void should_keep_answering_while_upgrading_tracker_comments()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto audio_path = test_files_folder().combine_file("Colorblind.mp3");
const auto module_path = test_files_folder().combine("retro").combine_file("giana.mod");
const auto scanned = core::date_t(2020, 1, 1, 0, 0, 0);
null_async_strategy as;
location_cache locations;
{
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
std::deque<item_db_write> writes;
for (const auto& path : {audio_path, module_path})
{
auto md = std::make_shared<prop::item_metadata>();
md->comment = "Triangle"_c;
item_db_write w;
w.path = path;
w.md = md;
w.metadata_scanned = scanned;
writes.emplace_back(std::move(w));
}
db.perform_writes(std::move(writes));
db.close();
}
{
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (open_result != SQLITE_OK ||
sqlite3_exec(raw.get(), "PRAGMA user_version = 6;", nullptr, nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception("Failed to set the tracker metadata version"s);
}
}
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto audio = index.find_item(audio_path);
const auto module = index.find_item(module_path);
const auto module_md = module.metadata.load();
assert_equal(scanned.to_int64(), audio.metadata_scanned.load().to_int64(), "other audio is not read again");
assert_equal(0ll, module.metadata_scanned.load().to_int64(), "a tracker module is");
assert_equal(true, module_md != nullptr && module_md->comment.sv() == "Triangle", "and answers meanwhile");
}
// Rolling a release back is ordinary, and the cache file is shared with whatever build the user
// goes back to. The rows stay readable, but the version stamp must come back down: left above this
// build's own, going forward again would find a version already satisfied and skip the upgrade it
// owed, trusting rows written in an older shape in the meantime.
static void should_reclaim_a_cache_written_by_a_newer_build()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto file_path = test_files_folder().combine_file("Test.jpg");
null_async_strategy as;
location_cache locations;
{
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
auto md = std::make_shared<prop::item_metadata>();
md->album = "test"_c;
md->dates.add(prop::date_source::exif_original, core::date_t(2012, 9, 14, 19, 21, 14));
std::deque<item_db_write> writes;
item_db_write w;
w.path = file_path;
w.md = md;
w.metadata_scanned = core::date_t(2020, 1, 1, 0, 0, 0);
writes.emplace_back(std::move(w));
db.perform_writes(std::move(writes));
db.close();
}
// Stamped by a release that does not exist yet.
{
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
if (open_result != SQLITE_OK ||
sqlite3_exec(raw.get(), "PRAGMA user_version = 99;", nullptr, nullptr, nullptr) != SQLITE_OK)
{
throw test_assert_exception("Failed to set the database metadata version"s);
}
}
{
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
const auto item = index.find_item(file_path);
const auto item_md = item.metadata.load();
assert_equal(true, item_md != nullptr, "the rows a newer build wrote are kept");
assert_equal(core::date_t(2012, 9, 14, 19, 21, 14), item_md->dates.original(), "and still answer");
assert_equal(core::date_t(2020, 1, 1, 0, 0, 0).to_int64(), item.metadata_scanned.load().to_int64(),
"and are not needlessly re-scanned");
db.close();
}
sqlite3* handle = nullptr;
const auto open_result = sqlite3_open(db_path.str().c_str(), &handle);
const std::unique_ptr<sqlite3, decltype(&sqlite3_close)> raw(handle, sqlite3_close);
auto stamped = -1;
if (open_result == SQLITE_OK)
{
sqlite3_stmt* stmt = nullptr;
if (sqlite3_prepare_v2(raw.get(), "PRAGMA user_version", -1, &stmt, nullptr) == SQLITE_OK)
{
if (sqlite3_step(stmt) == SQLITE_ROW) stamped = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
}
}
assert_equal(7, stamped, "the stamp comes back down to what this build writes");
}
// A cache file this build cannot read must be replaced, not refused. Everything it holds can be
// rebuilt by re-indexing, while failing to open it closes the app before the user can reach the
// reset that would repair it.
static void should_replace_an_unreadable_database()
{
const auto index_path = test_temps.next_path();
const auto db_path = core::file_path(index_path.folder(), index_path.file_name_without_extension(), ".db");
const auto file_path = test_files_folder().combine_file("Test.jpg");
{
std::ofstream corrupt(platform::to_stream_path(db_path), std::ios::binary | std::ios::trunc);
corrupt << "SQLite format 3\0not a database at all"s;
}
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
database db(index);
db.open(index_path.folder(), index_path.file_name_without_extension());
assert_equal(true, db.is_open(), "replaced database open");
assert_equal(false, db.has_errors(), "faults from the replaced file cleared");
const auto md = std::make_shared<prop::item_metadata>();
md->album = "replaced"_c;
std::deque<item_db_write> writes;
item_db_write w;
w.path = file_path;
w.md = md;
w.crc32c = 0x4321u;
writes.emplace_back(std::move(w));
db.perform_writes(std::move(writes));
db.load_index_values();
assert_equal(0x4321u, index.find_item(file_path).crc32c, "replaced database usable");
}
// Without a database the app still has to run. Every operation must complete rather than strand
// its caller, and the write queue must keep draining or the index would hoard encoded thumbnails
// for a whole session against a database that can never accept them.
static void should_run_without_a_database()
{
const auto missing_folder = test_temps.next_path().folder().combine("missing-cache-folder");
const auto file_path = test_files_folder().combine_file("Test.jpg");
null_async_strategy as;
const location_cache locations;
index_state index(as, locations);
database db(index);
db.open(missing_folder, "diffractor-cache");
assert_equal(false, db.is_open(), "database not open");
item_db_write w;
w.path = file_path;
w.md = std::make_shared<prop::item_metadata>();
index.db_writes().enqueue(std::move(w));
db.perform_writes();
assert_equal(true, index.db_writes().dequeue_all().empty(), "write queue drained");
assert_equal(true, db.load_item_imports().empty(), "no import history");
db.writes_item_imports({});
db.web_service_cache("key", "value");
assert_equal(true, db.web_service_cache("key").empty(), "no web cache");
db.clean({file_path});
assert_equal(false, core::is_valid(db.load_thumbnail(file_path).thumb), "no thumbnail");
}
// The scan loops hand their rows to the database in groups. One row at a time found the write queue
// empty every time, so it woke the database thread - and opened a transaction - once per file.
static void should_hand_scan_results_to_the_database_in_groups()
{
// Stands in for the database worker, which drains the whole write queue on every pass. That eager
// drain is what makes a per-row producer wake it again for the very next row.
class draining_async_strategy final : public null_async_strategy
{
public:
index_state* index = nullptr;
int drains = 0;
size_t rows = 0;
void queue_database(std::function<void(database&)> /*f*/) override
{
++drains;
rows += index->db_writes().dequeue_all().size();
}
};
draining_async_strategy as;
const location_cache locations;
index_state index(as, locations);
as.index = &index;
core::index_roots paths;
paths.folders.emplace(test_files_folder());
paths.excludes.emplace(test_files_folder().combine("excluded1"));
paths.exclude_wildcards.emplace("exclud*2"_c);
index.index_roots(paths);
index.index_folders(test_token);
index.scan_uncached(test_token);
as.rows += index.db_writes().dequeue_all().size();
// Without the fixtures actually being scanned the comparison below would pass vacuously.
assert_equal(expected_cached_item_count, index.stats.media_item_count, "cached item count");
assert_equal(true, as.rows >= 40, std::format("rows written: {}", as.rows));
// One drain per row is the defect. Grouping cannot need more than one per 64-row group.
assert_equal(true, as.drains <= static_cast<int>(as.rows / 8),
std::format("database drains {} for {} rows", as.drains, as.rows));
}
static void should_pack_item_properties()
{
const auto file_path = test_files_folder().combine_file("Test.jpg");
const auto md = extract_properties(file_path);
md->album = "test"_c;
md->orientation = core::orientation::bottom_right;
metadata_packer packer;
packer.pack(md);
const auto unpacked = std::make_shared<prop::item_metadata>();
metadata_unpacker unpacker(packer.cdata());
unpacker.unpack(unpacked);
assert_metadata(*md, *unpacked, "index");
assert_equal(md->orientation, unpacked->orientation, "index orientation");
md->audio_sample_rate = 192000;
md->audio_channels = 6;
metadata_packer audio_packer;
audio_packer.pack(md);