#include "backend.h"
#include "cluster.h"

#include <algorithm>
#include <atomic>
#include <iostream>
#include <sstream>
#include <unordered_set>

#ifdef NDEBUG
#define DBG_LOG(x) do {} while(0)
#else
#define DBG_LOG(x) do { std::cerr << x; } while(0)
#endif

namespace mediadb {

// Zero-copy read-only streambuf over a contiguous byte buffer
class MemBuf : public std::streambuf {
public:
    MemBuf(const std::uint8_t* data, std::size_t size) {
        auto* p = const_cast<char*>(reinterpret_cast<const char*>(data));
        setg(p, p, p + size);
    }
};

// ---- BinDb ----

BinDb::BinDb(const std::string& db_dir) : db_dir_(db_dir), memory_only_(false) {
    fs::create_directories(db_dir_);
    load();
}

BinDb::BinDb() : memory_only_(true) {}

fs::path BinDb::index_file() const { return fs::path(db_dir_) / "index.mdb"; }
fs::path BinDb::store_file(const std::string& store_id) const { return fs::path(db_dir_) / (store_id + ".mdb"); }

std::string BinDb::detect_kind(std::string ext) {
    std::transform(ext.begin(), ext.end(), ext.begin(),
                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
    if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" || ext == ".bmp" || ext == ".gif" || ext == ".avif") return "image";
    if (ext == ".mp4" || ext == ".mov" || ext == ".mkv" || ext == ".webm" || ext == ".avi" || ext == ".m4v" || ext == ".ts" || ext == ".flv") return "video";
    if (ext == ".mp3" || ext == ".flac" || ext == ".wav" || ext == ".ogg" || ext == ".aac" || ext == ".m4a" || ext == ".opus") return "audio";
    return "unknown";
}

// ---- stores ----

StoreRecord BinDb::create_store(const std::string& name) {
    StoreRecord s;
    {
        std::unique_lock lock(mutex_);
        s.id = make_uuid();
        s.name = name;
        s.created_at = now_iso8601();
        stores_[s.id] = s;
        save_index_locked();
        ensure_store_file_locked(s.id);
    }
    return s;
}

std::optional<StoreRecord> BinDb::get_store(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(id);
    if (it == stores_.end()) return std::nullopt;
    return it->second;
}

std::vector<StoreRecord> BinDb::list_stores() const {
    std::shared_lock lock(mutex_);
    std::vector<StoreRecord> out;
    out.reserve(stores_.size());
    for (const auto& [k, v] : stores_) out.push_back(v);
    return out;
}

bool BinDb::delete_store(const std::string& id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(id);
        if (it == stores_.end()) return false;
        for (const auto& aid : it->second.album_ids) {
            auto ait = albums_.find(aid);
            if (ait != albums_.end()) {
                for (const auto& mid : ait->second.media_ids) media_.erase(mid);
                albums_.erase(ait);
            }
        }
        mmaps_.erase(id);
        if (!memory_only_) {
            auto sf = store_file(id);
            if (fs::exists(sf)) fs::remove(sf);
        }
        stores_.erase(it);
        if (!memory_only_) save_index_locked();
    }
    return true;
}

bool BinDb::rename_store(const std::string& id, const std::string& new_name) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(id);
        if (it == stores_.end()) return false;
        it->second.name = new_name;
        save_index_locked();
    }
    return true;
}

// ---- store auth domains ----

std::string BinDb::add_store_domain(const std::string& store_id, AuthDomain domain) {
    std::string did;
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return {};
        domain.id = make_uuid();
        did = domain.id;
        it->second.auth_domains.push_back(std::move(domain));
        save_index_locked();
    }
    return did;
}

bool BinDb::remove_store_domain(const std::string& store_id, const std::string& domain_id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return false;
        auto& domains = it->second.auth_domains;
        auto dit = std::find_if(domains.begin(), domains.end(),
                                [&](const AuthDomain& d) { return d.id == domain_id; });
        if (dit == domains.end()) return false;
        domains.erase(dit);
        save_index_locked();
    }
    return true;
}

std::vector<AuthDomain> BinDb::list_store_domains(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(store_id);
    if (it == stores_.end()) return {};
    return it->second.auth_domains;
}

std::optional<AuthDomain> BinDb::find_domain_by_label(const std::string& label) const {
    std::shared_lock lock(mutex_);
    for (const auto& [sid, store] : stores_) {
        for (const auto& d : store.auth_domains) {
            if (d.label == label) return d;
        }
    }
    return std::nullopt;
}

// ---- store ACLs ----

std::string BinDb::add_store_acl(const std::string& store_id, StoreACL acl) {
    std::string aid;
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return {};
        acl.id = make_uuid();
        aid = acl.id;
        it->second.acls.push_back(std::move(acl));
        save_index_locked();
    }
    return aid;
}

bool BinDb::remove_store_acl(const std::string& store_id, const std::string& acl_id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return false;
        auto& acls = it->second.acls;
        auto ait = std::find_if(acls.begin(), acls.end(),
                                [&](const StoreACL& a) { return a.id == acl_id; });
        if (ait == acls.end()) return false;
        acls.erase(ait);
        save_index_locked();
    }
    return true;
}

std::vector<StoreACL> BinDb::list_store_acls(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(store_id);
    if (it == stores_.end()) return {};
    return it->second.acls;
}

std::vector<std::string> BinDb::all_acl_group_ids() const {
    std::shared_lock lock(mutex_);
    std::set<std::string> ids;
    for (const auto& [sid, s] : stores_) {
        for (const auto& acl : s.acls) {
            ids.insert(acl.group_id);
        }
    }
    return {ids.begin(), ids.end()};
}

// ---- albums ----

std::optional<AlbumRecord> BinDb::create_album(const std::string& store_id, const std::string& name) {
    std::optional<AlbumRecord> result;
    {
        std::unique_lock lock(mutex_);
        auto sit = stores_.find(store_id);
        if (sit == stores_.end()) return std::nullopt;

        AlbumRecord a;
        a.id = make_uuid();
        a.name = name;
        a.store_id = store_id;
        a.created_at = now_iso8601();
        albums_[a.id] = a;
        sit->second.album_ids.push_back(a.id);
        append_album_locked(store_id, albums_[a.id]);
        result = albums_[a.id];
    }
    return result;
}

std::optional<AlbumRecord> BinDb::get_album(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = albums_.find(id);
    if (it == albums_.end()) return std::nullopt;
    return it->second;
}

std::vector<AlbumRecord> BinDb::list_albums(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    std::vector<AlbumRecord> out;
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return out;
    for (const auto& aid : sit->second.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) out.push_back(ait->second);
    }
    return out;
}

bool BinDb::delete_album(const std::string& id) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto ait = albums_.find(id);
        if (ait == albums_.end()) return false;
        store_id = ait->second.store_id;
        auto sit = stores_.find(store_id);
        if (sit != stores_.end()) {
            auto& aids = sit->second.album_ids;
            aids.erase(std::remove(aids.begin(), aids.end(), id), aids.end());
        }
        // Zero all media records belonging to this album
        for (const auto& mid : ait->second.media_ids) {
            auto mit2 = media_.find(mid);
            if (mit2 != media_.end()) {
                zero_record_locked(store_id, mit2->second.record_offset);
                media_.erase(mit2);
            }
        }
        // Zero the album record itself
        zero_record_locked(store_id, ait->second.record_offset);
        albums_.erase(ait);
    }
    return true;
}

bool BinDb::set_album_public(const std::string& album_id, bool is_public) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto it = albums_.find(album_id);
        if (it == albums_.end()) return false;
        // Zero old album record, update flag, append new one
        zero_record_locked(it->second.store_id, it->second.record_offset);
        it->second.is_public = is_public;
        store_id = it->second.store_id;
        append_album_locked(store_id, it->second);
    }
    return true;
}

bool BinDb::is_album_public(const std::string& album_id) const {
    std::shared_lock lock(mutex_);
    auto it = albums_.find(album_id);
    if (it == albums_.end()) return false;
    return it->second.is_public;
}

bool BinDb::is_media_public(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return false;
    auto ait = albums_.find(it->second.album_id);
    if (ait == albums_.end()) return false;
    return ait->second.is_public;
}

std::vector<AlbumRecord> BinDb::list_public_albums() const {
    std::shared_lock lock(mutex_);
    std::vector<AlbumRecord> out;
    for (const auto& [id, album] : albums_) {
        if (album.is_public) out.push_back(album);
    }
    return out;
}

// ---- media ----

std::optional<MediaRecord> BinDb::add_media(const std::string& album_id,
                                             const std::string& filename,
                                             const std::string& content_type,
                                             std::vector<std::uint8_t> data) {
    std::optional<MediaRecord> result;
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto ait = albums_.find(album_id);
        if (ait == albums_.end()) return std::nullopt;

        store_id = ait->second.store_id;
        const fs::path p(filename);
        MediaRecord m;
        m.id = make_uuid();
        m.original_filename = filename;
        m.media_kind = detect_kind(p.extension().string());
        m.content_type = content_type;
        m.album_id = album_id;
        m.store_id = store_id;
        m.created_at = now_iso8601();
        m.size_bytes = data.size();
        m.pending_data = std::move(data);
        m.data_offset = 0;
        const std::string mid = m.id;
        media_[mid] = std::move(m);
        ait->second.media_ids.push_back(mid);

        append_media_locked(store_id, media_[mid]);
        result = media_[mid];
    }
    return result;
}

std::optional<MediaRecord> BinDb::get_media(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(id);
    if (it == media_.end()) return std::nullopt;
    return it->second;
}

std::vector<MediaRecord> BinDb::list_media(const std::string& album_id) const {
    std::shared_lock lock(mutex_);
    std::vector<MediaRecord> out;
    auto ait = albums_.find(album_id);
    if (ait == albums_.end()) return out;
    for (const auto& mid : ait->second.media_ids) {
        auto mit = media_.find(mid);
        if (mit != media_.end()) out.push_back(mit->second);
    }
    return out;
}

bool BinDb::delete_media(const std::string& id) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto mit = media_.find(id);
        if (mit == media_.end()) return false;
        store_id = mit->second.store_id;
        auto rec_off = mit->second.record_offset;
        auto ait = albums_.find(mit->second.album_id);
        if (ait != albums_.end()) {
            auto& mids = ait->second.media_ids;
            mids.erase(std::remove(mids.begin(), mids.end(), id), mids.end());
        }
        media_.erase(mit);
        zero_record_locked(store_id, rec_off);
    }
    return true;
}

// ---- read raw media data ----

std::vector<std::uint8_t> BinDb::read_media_data(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return {};
    const auto& m = it->second;
    if (m.size_bytes == 0) return {};
    if (!m.pending_data.empty()) return m.pending_data;

    auto mit = mmaps_.find(m.store_id);
    if (mit != mmaps_.end() && mit->second.valid()) {
        const auto& mm = mit->second;
        if (m.data_offset + m.size_bytes <= mm.size) {
            const auto* ptr = mm.data() + m.data_offset;
            return std::vector<std::uint8_t>(ptr, ptr + m.size_bytes);
        }
    }
    std::ifstream in(store_file(m.store_id), std::ios::binary);
    if (!in.is_open()) return {};
    in.seekg(static_cast<std::streamoff>(m.data_offset));
    std::vector<std::uint8_t> buf(m.size_bytes);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(m.size_bytes));
    return buf;
}

std::vector<std::uint8_t> BinDb::read_media_data_range(const std::string& media_id,
                                                         std::uint64_t offset,
                                                         std::uint64_t length) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return {};
    const auto& m = it->second;
    if (m.size_bytes == 0 || offset >= m.size_bytes) return {};
    const std::uint64_t avail = m.size_bytes - offset;
    const std::uint64_t to_read = std::min(length, avail);

    if (!m.pending_data.empty()) {
        return std::vector<std::uint8_t>(
            m.pending_data.begin() + static_cast<std::ptrdiff_t>(offset),
            m.pending_data.begin() + static_cast<std::ptrdiff_t>(offset + to_read));
    }

    auto mit = mmaps_.find(m.store_id);
    if (mit != mmaps_.end() && mit->second.valid()) {
        const auto& mm = mit->second;
        const auto abs_off = m.data_offset + offset;
        if (abs_off + to_read <= mm.size) {
            const auto* ptr = mm.data() + abs_off;
            return std::vector<std::uint8_t>(ptr, ptr + to_read);
        }
    }
    std::ifstream in(store_file(m.store_id), std::ios::binary);
    if (!in.is_open()) return {};
    in.seekg(static_cast<std::streamoff>(m.data_offset + offset));
    std::vector<std::uint8_t> buf(to_read);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(to_read));
    return buf;
}

std::uint64_t BinDb::get_media_size(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return 0;
    return it->second.size_bytes;
}

// ---- export ----

bool BinDb::export_db(const std::string& path) const {
    std::shared_lock lock(mutex_);
    const std::string tmp = path + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return false;
        out.write(EXPORT_MAGIC_V2, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(stores_.size()));

        for (const auto& [sid, store] : stores_) {
            bin_write_str(out, store.id);
            bin_write_str(out, store.name);
            bin_write_str(out, store.created_at);
            bin_write_u32(out, static_cast<std::uint32_t>(store.album_ids.size()));

            for (const auto& aid : store.album_ids) {
                auto ait = albums_.find(aid);
                if (ait == albums_.end()) {
                    bin_write_str(out, aid);
                    bin_write_str(out, "");
                    bin_write_str(out, "");
                    { std::uint8_t pub = 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
                    bin_write_u32(out, 0);
                    continue;
                }
                const auto& album = ait->second;
                bin_write_str(out, album.id);
                bin_write_str(out, album.name);
                bin_write_str(out, album.created_at);
                { std::uint8_t pub = album.is_public ? 1 : 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
                bin_write_u32(out, static_cast<std::uint32_t>(album.media_ids.size()));

                for (const auto& mid : album.media_ids) {
                    auto mit = media_.find(mid);
                    if (mit == media_.end()) continue;
                    const auto& m = mit->second;

                    bin_write_str(out, m.id);
                    bin_write_str(out, m.original_filename);
                    bin_write_str(out, m.media_kind);
                    bin_write_str(out, m.content_type);
                    bin_write_str(out, m.created_at);
                    bin_write_u64(out, m.size_bytes);

                    if (!m.pending_data.empty()) {
                        out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                                  static_cast<std::streamsize>(m.pending_data.size()));
                    } else if (m.data_offset > 0 && m.size_bytes > 0) {
                        std::ifstream sin(store_file(sid), std::ios::binary);
                        if (sin.is_open()) {
                            sin.seekg(static_cast<std::streamoff>(m.data_offset));
                            constexpr std::size_t BUF = 1024 * 1024;
                            std::vector<char> buf(BUF);
                            std::uint64_t remaining = m.size_bytes;
                            while (remaining > 0 && sin.good()) {
                                auto chunk = static_cast<std::streamsize>(
                                    std::min<std::uint64_t>(remaining, BUF));
                                sin.read(buf.data(), chunk);
                                auto got = sin.gcount();
                                if (got > 0) out.write(buf.data(), got);
                                remaining -= static_cast<std::uint64_t>(got);
                                if (got < chunk) break;
                            }
                        }
                    }
                }
            }
        }
    }
    fs::rename(tmp, path);
    return true;
}

std::vector<std::uint8_t> BinDb::export_db_to_buffer() const {
    std::shared_lock lock(mutex_);
    std::ostringstream oss(std::ios::binary);
    oss.write(EXPORT_MAGIC_V2, 4);

    auto wr_u32 = [&](std::uint32_t v) { oss.write(reinterpret_cast<const char*>(&v), 4); };
    auto wr_u64 = [&](std::uint64_t v) { oss.write(reinterpret_cast<const char*>(&v), 8); };
    auto wr_str = [&](const std::string& s) {
        auto len = static_cast<std::uint32_t>(s.size());
        wr_u32(len);
        if (len > 0) oss.write(s.data(), len);
    };

    wr_u32(static_cast<std::uint32_t>(stores_.size()));

    for (const auto& [sid, store] : stores_) {
        wr_str(store.id);
        wr_str(store.name);
        wr_str(store.created_at);
        wr_u32(static_cast<std::uint32_t>(store.album_ids.size()));

        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) {
                wr_str(aid);
                wr_str("");
                wr_str("");
                { std::uint8_t pub = 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
                wr_u32(0);
                continue;
            }
            const auto& album = ait->second;
            wr_str(album.id);
            wr_str(album.name);
            wr_str(album.created_at);
            { std::uint8_t pub = album.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
            wr_u32(static_cast<std::uint32_t>(album.media_ids.size()));

            for (const auto& mid : album.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                const auto& m = mit->second;

                wr_str(m.id);
                wr_str(m.original_filename);
                wr_str(m.media_kind);
                wr_str(m.content_type);
                wr_str(m.created_at);
                wr_u64(m.size_bytes);

                if (!m.pending_data.empty()) {
                    oss.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                } else if (m.data_offset > 0 && m.size_bytes > 0) {
                    std::ifstream sin(store_file(sid), std::ios::binary);
                    if (sin.is_open()) {
                        sin.seekg(static_cast<std::streamoff>(m.data_offset));
                        constexpr std::size_t BUF = 1024 * 1024;
                        std::vector<char> buf(BUF);
                        std::uint64_t remaining = m.size_bytes;
                        while (remaining > 0 && sin.good()) {
                            auto chunk = static_cast<std::streamsize>(
                                std::min<std::uint64_t>(remaining, BUF));
                            sin.read(buf.data(), chunk);
                            auto got = sin.gcount();
                            if (got > 0) oss.write(buf.data(), got);
                            remaining -= static_cast<std::uint64_t>(got);
                            if (got < chunk) break;
                        }
                    }
                }
            }
        }
    }

    std::string s = std::move(oss).str();
    return std::vector<std::uint8_t>(std::make_move_iterator(s.begin()), std::make_move_iterator(s.end()));
}

// ---- import ----

bool BinDb::import_from_stream(std::istream& in) {
    return import_from_stream(in, nullptr);
}

bool BinDb::import_from_stream(std::istream& in,
                                std::function<void(const std::string&, const uint8_t*, size_t)> on_replicate) {
    std::unique_lock lock(mutex_);

    char magic[4]{};
    in.read(magic, 4);
    const bool import_v2 = (std::memcmp(magic, EXPORT_MAGIC_V2, 4) == 0);
    if (!import_v2 && std::memcmp(magic, EXPORT_MAGIC, 4) != 0) {
        std::cerr << "[IMPORT] bad magic\n";
        return false;
    }

    // When streaming to cluster (on_replicate set), build the store buffer
    // directly from the input stream without loading media into pending_data.
    // This avoids keeping all media bytes in memory at once.
    const bool stream_to_cluster = (on_replicate != nullptr);

    auto num_stores = bin_read_u32(in);
    DBG_LOG("[IMPORT] start: " << num_stores << " stores, v2=" << import_v2
              << ", cluster=" << stream_to_cluster << "\n");
    std::size_t total_media_imported = 0;

    // Reusable I/O buffer — allocated once, grown as needed
    std::vector<char> import_buf;
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        std::string store_id = bin_read_str(in);
        std::string store_name = bin_read_str(in);
        std::string store_created = bin_read_str(in);
        DBG_LOG("[IMPORT] store " << (si+1) << "/" << num_stores
                  << " id=" << store_id << " name=" << store_name << "\n");

        bool new_store = (stores_.find(store_id) == stores_.end());
        if (new_store) {
            StoreRecord s;
            s.id = store_id;
            s.name = store_name;
            s.created_at = store_created;
            stores_[store_id] = s;
        }

        // Ensure MDS3 store file exists and open it once for all appends
        std::ofstream store_out;
        if (!stream_to_cluster && !memory_only_) {
            ensure_store_file_locked(store_id);
            store_out.open(store_file(store_id), std::ios::binary | std::ios::app);
        }

        // When streaming, build the MDS2 store buffer on the fly
        std::ostringstream store_buf;
        if (stream_to_cluster) {
            store_buf.write(STORE_MAGIC_V2, 4);
            // Placeholder for album count — will be patched below
            bin_write_u32(store_buf, 0);
        }
        std::uint32_t album_count = 0;

        auto num_albums = bin_read_u32(in);
        DBG_LOG("[IMPORT]   " << num_albums << " albums in store\n");
        for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
            std::string album_id = bin_read_str(in);
            std::string album_name = bin_read_str(in);
            std::string album_created = bin_read_str(in);

            bool album_is_public = false;
            if (import_v2) {
                std::uint8_t pub = 0;
                in.read(reinterpret_cast<char*>(&pub), 1);
                album_is_public = (pub != 0);
            }

            bool new_album = (albums_.find(album_id) == albums_.end());
            if (new_album) {
                AlbumRecord a;
                a.id = album_id;
                a.name = album_name;
                a.store_id = store_id;
                a.created_at = album_created;
                a.is_public = album_is_public;
                albums_[album_id] = a;
                stores_[store_id].album_ids.push_back(album_id);
                if (store_out.is_open()) {
                    // Write album record directly to open store file
                    std::ostringstream abody(std::ios::binary);
                    bin_write_str(abody, a.id);
                    bin_write_str(abody, a.name);
                    bin_write_str(abody, a.created_at);
                    { std::uint8_t pub = a.is_public ? 1 : 0;
                      abody.write(reinterpret_cast<const char*>(&pub), 1); }
                    std::string abody_str = abody.str();
                    auto aoff = static_cast<std::uint64_t>(store_out.tellp());
                    store_out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
                    auto absz = static_cast<std::uint64_t>(abody_str.size());
                    store_out.write(reinterpret_cast<const char*>(&absz), 8);
                    store_out.write(abody_str.data(), static_cast<std::streamsize>(abody_str.size()));
                    albums_[album_id].record_offset = aoff;
                }
            }

            // Start writing album to store buffer
            if (stream_to_cluster) {
                bin_write_str(store_buf, album_id);
                bin_write_str(store_buf, album_name);
                bin_write_str(store_buf, album_created);
                { std::uint8_t pub = album_is_public ? 1 : 0;
                  store_buf.write(reinterpret_cast<const char*>(&pub), 1); }
            }

            auto num_media = bin_read_u32(in);
            DBG_LOG("[IMPORT]   album " << (ai+1) << "/" << num_albums
                      << " \"" << album_name << "\" media=" << num_media << "\n");

            if (stream_to_cluster) {
                bin_write_u32(store_buf, num_media);
            }

            for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
                std::string media_id = bin_read_str(in);
                std::string orig_filename = bin_read_str(in);
                std::string media_kind = bin_read_str(in);
                std::string content_type = bin_read_str(in);
                std::string created_at = bin_read_str(in);
                std::uint64_t data_size = bin_read_u64(in);

                bool new_media = (media_.find(media_id) == media_.end());
                if ((mi + 1) % 50 == 0 || mi == 0 || mi + 1 == num_media) {
                    DBG_LOG("[IMPORT]     media " << (mi+1) << "/" << num_media
                              << " size=" << data_size << " new=" << new_media
                              << " stream_pos=" << in.tellg() << "\n");
                }

                if (stream_to_cluster) {
                    // Write media metadata to store buffer (NO raw data — stored separately)
                    bin_write_str(store_buf, media_id);
                    bin_write_str(store_buf, orig_filename);
                    bin_write_str(store_buf, media_kind);
                    bin_write_str(store_buf, content_type);
                    bin_write_str(store_buf, created_at);
                    bin_write_u64(store_buf, data_size);

                    if (data_size > 0 && new_media) {
                        // Read media data and replicate as separate key: media:<id>
                        std::vector<std::uint8_t> media_data(data_size);
                        in.read(reinterpret_cast<char*>(media_data.data()),
                                static_cast<std::streamsize>(data_size));
                        auto got = static_cast<std::size_t>(in.gcount());
                        if (got < data_size) media_data.resize(got);
                        on_replicate("media:" + media_id,
                                     media_data.data(), media_data.size());
                    } else if (data_size > 0) {
                        // Existing media: skip past raw data
                        in.seekg(static_cast<std::streamoff>(data_size), std::ios::cur);
                    }

                    // Register metadata only (no raw data in RAM)
                    if (new_media) {
                        MediaRecord m;
                        m.id = media_id;
                        m.original_filename = orig_filename;
                        m.media_kind = media_kind;
                        m.content_type = content_type;
                        m.created_at = created_at;
                        m.size_bytes = data_size;
                        m.album_id = album_id;
                        m.store_id = store_id;
                        media_[media_id] = std::move(m);
                        albums_[album_id].media_ids.push_back(media_id);
                        ++total_media_imported;
                    }
                } else {
                    // Non-cluster import: stream directly to MDS3 store file
                    if (new_media) {
                        MediaRecord m;
                        m.id = media_id;
                        m.original_filename = orig_filename;
                        m.media_kind = media_kind;
                        m.content_type = content_type;
                        m.created_at = created_at;
                        m.size_bytes = data_size;
                        m.album_id = album_id;
                        m.store_id = store_id;

                        // Compute metadata size without ostringstream
                        auto meta_size = static_cast<std::uint64_t>(
                            4 + m.id.size() + 4 + m.album_id.size() +
                            4 + m.original_filename.size() + 4 + m.media_kind.size() +
                            4 + m.content_type.size() + 4 + m.created_at.size() + 8);
                        auto total_body = meta_size + data_size;

                        // Write record header
                        m.record_offset = static_cast<std::uint64_t>(store_out.tellp());
                        store_out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
                        store_out.write(reinterpret_cast<const char*>(&total_body), 8);

                        // Write metadata fields directly
                        bin_write_str(store_out, m.id);
                        bin_write_str(store_out, m.album_id);
                        bin_write_str(store_out, m.original_filename);
                        bin_write_str(store_out, m.media_kind);
                        bin_write_str(store_out, m.content_type);
                        bin_write_str(store_out, m.created_at);
                        bin_write_u64(store_out, m.size_bytes);
                        m.data_offset = static_cast<std::uint64_t>(store_out.tellp());

                        // Stream raw data directly: input → store file (no RAM buffer)
                        if (data_size > 0) {
                            constexpr std::size_t BUF = 1024 * 1024;
                            if (import_buf.size() < std::min<std::size_t>(BUF, data_size))
                                import_buf.resize(std::min<std::size_t>(BUF, data_size));
                            std::uint64_t remaining = data_size;
                            while (remaining > 0 && in.good()) {
                                auto chunk = static_cast<std::streamsize>(
                                    std::min<std::uint64_t>(remaining, import_buf.size()));
                                in.read(import_buf.data(), chunk);
                                auto got = in.gcount();
                                if (got > 0) store_out.write(import_buf.data(), got);
                                remaining -= static_cast<std::uint64_t>(got);
                                if (got < chunk) break;
                            }
                        }

                        media_[media_id] = std::move(m);
                        albums_[album_id].media_ids.push_back(media_id);
                        ++total_media_imported;
                    } else {
                        if (data_size > 0) {
                            in.seekg(static_cast<std::streamoff>(data_size), std::ios::cur);
                        }
                    }
                }
            }

            album_count++;
        }

        // Non-cluster: close store file, then mmap
        if (store_out.is_open()) {
            store_out.close();
            mmap_store(store_id);
        }

        // Replicate metadata-only store buffer to cluster
        if (stream_to_cluster) {
            // Patch album count at offset 4 (after STORE_MAGIC_V2)
            std::string buf_str = store_buf.str();
            std::memcpy(buf_str.data() + 4, &album_count, 4);
            DBG_LOG("[IMPORT] replicate store:" << store_id
                      << " buf_size=" << buf_str.size() << "\n");
            on_replicate("store:" + store_id,
                         reinterpret_cast<const uint8_t*>(buf_str.data()),
                         buf_str.size());
            DBG_LOG("[IMPORT] replicate store:" << store_id << " done\n");

            // Save and replicate index after each store for crash consistency.
            // If import is interrupted, stores already replicated will have
            // a matching index referencing them.
            save_index_locked();
            auto idx_buf = save_index_to_buffer_locked();
            if (!idx_buf.empty()) {
                DBG_LOG("[IMPORT] replicate index size=" << idx_buf.size() << "\n");
                on_replicate("index", idx_buf.data(), idx_buf.size());
                DBG_LOG("[IMPORT] replicate index done\n");
            }
        }
    }

    // Vacuum touched stores to compact partial/duplicate records
    // from previous failed imports that used ios::app.
    if (!stream_to_cluster && !memory_only_) {
        for (const auto& [sid, _] : stores_)
            vacuum_store_locked(sid);
    }

    // Save index once after all stores are imported (not per-store)
    save_index_locked();

    // Replicate final index to cluster
    if (on_replicate) {
        auto buf = save_index_to_buffer_locked();
        if (!buf.empty()) {
            on_replicate("index", buf.data(), buf.size());
        }
    }

    return true;
}

bool BinDb::import_db(const std::string& path) {
    std::ifstream in(path, std::ios::binary);
    if (!in.is_open()) return false;
    return import_from_stream(in);
}

bool BinDb::import_db_from_buffer(const std::vector<std::uint8_t>& data) {
    return import_db_from_buffer(data.data(), data.size());
}

bool BinDb::import_db_from_buffer(const std::uint8_t* data, std::size_t len) {
    MemBuf mbuf(data, len);
    std::istream in(&mbuf);
    return import_from_stream(in);
}

bool BinDb::import_db_from_buffer(const std::uint8_t* data, std::size_t len,
                                   std::function<void(const std::string&, const uint8_t*, size_t)> on_replicate) {
    MemBuf mbuf(data, len);
    std::istream in(&mbuf);
    return import_from_stream(in, std::move(on_replicate));
}

// ---- Incremental import session (state machine fed with network chunks) ----

class BinDbImportSession : public ImportSession {
public:
    BinDbImportSession(BinDb& db,
                       ReplicateFn on_replicate,
                       BeginReplicateFn begin_replicate,
                       BinDb::StoreLifecycleFn on_store_begin = nullptr,
                       BinDb::StoreLifecycleFn on_store_end = nullptr)
        : db_(db), on_replicate_(std::move(on_replicate)),
          begin_replicate_(std::move(begin_replicate)),
          on_store_begin_(std::move(on_store_begin)),
          on_store_end_(std::move(on_store_end)),
          stream_to_cluster_(on_replicate_ != nullptr) {}

    ~BinDbImportSession() override {
        if (!done_.load(std::memory_order_acquire)) {
            // Aborted — close any open file
            if (store_out_.is_open()) store_out_.close();
        }
    }

    void feed(const char* data, size_t len) override {
        if (done_.load(std::memory_order_acquire)) return;
        {
            std::unique_lock lock(db_.mutex_);
            buf_.insert(buf_.end(), data, data + len);
            total_fed_ += len;
            while (!done_.load(std::memory_order_relaxed) && try_advance()) {}
            // Compact consumed bytes
            if (pos_ > 0) {
                buf_.erase(buf_.begin(), buf_.begin() + static_cast<std::ptrdiff_t>(pos_));
                pos_ = 0;
            }
        }
        // Execute deferred ops inline (for callers that don't use take_deferred)
        drain_deferred();
    }

    std::vector<std::function<void()>> take_deferred() override {
        std::lock_guard<std::mutex> lk(deferred_mutex_);
        std::vector<std::function<void()>> out;
        out.swap(pending_ops_);
        return out;
    }

    bool done() const override {
        return done_.load(std::memory_order_acquire);
    }
    bool ok() const override { return ok_.load(std::memory_order_acquire); }
    std::string error_message() const override {
        std::shared_lock lock(error_mutex_);
        return error_msg_;
    }

private:
    // ---- read helpers (return false if not enough data) ----

    bool try_read_bytes(void* dest, size_t n) {
        if (buf_.size() - pos_ < n) return false;
        std::memcpy(dest, buf_.data() + pos_, n);
        pos_ += n;
        return true;
    }

    bool try_read_u32(std::uint32_t& out) { return try_read_bytes(&out, 4); }
    bool try_read_u64(std::uint64_t& out) { return try_read_bytes(&out, 8); }

    bool try_read_str(std::string& out) {
        if (buf_.size() - pos_ < 4) return false;
        std::uint32_t len;
        std::memcpy(&len, buf_.data() + pos_, 4);
        if (buf_.size() - pos_ < 4 + len) return false;
        out.assign(buf_.data() + pos_ + 4, len);
        pos_ += 4 + len;
        return true;
    }

    // ---- state machine ----

    bool try_advance() {
        switch (phase_) {

        case Phase::MAGIC: {
            char magic[4];
            if (!try_read_bytes(magic, 4)) return false;
            is_v2_ = (std::memcmp(magic, BinDb::EXPORT_MAGIC_V2, 4) == 0);
            if (!is_v2_ && std::memcmp(magic, BinDb::EXPORT_MAGIC, 4) != 0) {
                done_.store(true, std::memory_order_release);
                return false;
            }
            phase_ = Phase::NUM_STORES;
            return true;
        }

        case Phase::NUM_STORES: {
            if (!try_read_u32(num_stores_)) return false;
            si_ = 0;
            if (num_stores_ == 0) { finish_import(); return false; }
            phase_ = Phase::STORE_HEADER;
            return true;
        }

        case Phase::STORE_HEADER: {
            size_t saved = pos_;
            if (!try_read_str(cur_store_id_) ||
                !try_read_str(cur_store_name_) ||
                !try_read_str(cur_store_created_)) {
                pos_ = saved;
                return false;
            }
            // Create store if new
            if (db_.stores_.find(cur_store_id_) == db_.stores_.end()) {
                StoreRecord s;
                s.id = cur_store_id_;
                s.name = cur_store_name_;
                s.created_at = cur_store_created_;
                db_.stores_[cur_store_id_] = s;
            }
            // Open store file (non-cluster)
            if (!stream_to_cluster_ && !db_.memory_only_) {
                db_.ensure_store_file_locked(cur_store_id_);
                store_out_.open(db_.store_file(cur_store_id_),
                                std::ios::binary | std::ios::app);
                if (!store_out_.is_open()) {
                    fail("cannot open store file: " + db_.store_file(cur_store_id_).string());
                    return false;
                }
            }
            touched_stores_.insert(cur_store_id_);
            // Cluster: start store metadata buffer
            if (stream_to_cluster_) {
                if (on_store_begin_) {
                    std::lock_guard<std::mutex> dlk(deferred_mutex_);
                    pending_ops_.push_back(on_store_begin_);
                }
                store_meta_buf_.str("");
                store_meta_buf_.clear();
                store_meta_buf_.write(BinDb::STORE_MAGIC_V2, 4);
                bin_write_u32(store_meta_buf_, 0); // placeholder for album count
                store_album_count_ = 0;
            }
            phase_ = Phase::NUM_ALBUMS;
            return true;
        }

        case Phase::NUM_ALBUMS: {
            if (!try_read_u32(num_albums_)) return false;
            ai_ = 0;
            if (num_albums_ == 0) { finish_store(); return true; }
            phase_ = Phase::ALBUM_HEADER;
            return true;
        }

        case Phase::ALBUM_HEADER: {
            size_t saved = pos_;
            std::string album_id, album_name, album_created;
            if (!try_read_str(album_id) ||
                !try_read_str(album_name) ||
                !try_read_str(album_created)) {
                pos_ = saved;
                return false;
            }
            bool album_is_public = false;
            if (is_v2_) {
                std::uint8_t pub = 0;
                if (!try_read_bytes(&pub, 1)) { pos_ = saved; return false; }
                album_is_public = (pub != 0);
            }
            cur_album_id_ = album_id;
            bool new_album = (db_.albums_.find(album_id) == db_.albums_.end());
            if (new_album) {
                AlbumRecord a;
                a.id = album_id;
                a.name = album_name;
                a.store_id = cur_store_id_;
                a.created_at = album_created;
                a.is_public = album_is_public;
                db_.albums_[album_id] = a;
                db_.stores_[cur_store_id_].album_ids.push_back(album_id);
                // Write album record to store file
                if (store_out_.is_open()) {
                    std::ostringstream abody(std::ios::binary);
                    bin_write_str(abody, a.id);
                    bin_write_str(abody, a.name);
                    bin_write_str(abody, a.created_at);
                    { std::uint8_t pub = a.is_public ? 1 : 0;
                      abody.write(reinterpret_cast<const char*>(&pub), 1); }
                    std::string abody_str = abody.str();
                    auto aoff = static_cast<std::uint64_t>(store_out_.tellp());
                    store_out_.write(reinterpret_cast<const char*>(&BinDb::REC_ALBUM), 1);
                    auto absz = static_cast<std::uint64_t>(abody_str.size());
                    store_out_.write(reinterpret_cast<const char*>(&absz), 8);
                    store_out_.write(abody_str.data(), static_cast<std::streamsize>(abody_str.size()));
                    if (!store_out_.good()) {
                        fail("write error on store file for album " + album_id);
                        return false;
                    }
                    db_.albums_[album_id].record_offset = aoff;
                }
            }
            // Cluster: write album metadata
            if (stream_to_cluster_) {
                bin_write_str(store_meta_buf_, album_id);
                bin_write_str(store_meta_buf_, album_name);
                bin_write_str(store_meta_buf_, album_created);
                { std::uint8_t pub = album_is_public ? 1 : 0;
                  store_meta_buf_.write(reinterpret_cast<const char*>(&pub), 1); }
                ++store_album_count_;
            }
            phase_ = Phase::NUM_MEDIA;
            return true;
        }

        case Phase::NUM_MEDIA: {
            if (!try_read_u32(num_media_)) return false;
            if (stream_to_cluster_) bin_write_u32(store_meta_buf_, num_media_);
            mi_ = 0;
            if (num_media_ == 0) { advance_after_album(); return true; }
            phase_ = Phase::MEDIA_HEADER;
            return true;
        }

        case Phase::MEDIA_HEADER: {
            size_t saved = pos_;
            std::string media_id, orig_filename, media_kind, content_type, created_at;
            std::uint64_t data_size;
            if (!try_read_str(media_id) ||
                !try_read_str(orig_filename) ||
                !try_read_str(media_kind) ||
                !try_read_str(content_type) ||
                !try_read_str(created_at) ||
                !try_read_u64(data_size)) {
                pos_ = saved;
                return false;
            }
            cur_media_id_ = media_id;
            cur_media_new_ = (db_.media_.find(media_id) == db_.media_.end());
            data_size_ = data_size;
            data_written_ = 0;

            // Cluster: write media metadata to store buffer
            if (stream_to_cluster_) {
                bin_write_str(store_meta_buf_, media_id);
                bin_write_str(store_meta_buf_, orig_filename);
                bin_write_str(store_meta_buf_, media_kind);
                bin_write_str(store_meta_buf_, content_type);
                bin_write_str(store_meta_buf_, created_at);
                bin_write_u64(store_meta_buf_, data_size);
            }

            // Non-cluster: write media header to store file
            if (!stream_to_cluster_ && cur_media_new_ && store_out_.is_open()) {
                auto meta_size = static_cast<std::uint64_t>(
                    4 + media_id.size() + 4 + cur_album_id_.size() +
                    4 + orig_filename.size() + 4 + media_kind.size() +
                    4 + content_type.size() + 4 + created_at.size() + 8);
                auto total_body = meta_size + data_size;
                media_record_offset_ = static_cast<std::uint64_t>(store_out_.tellp());
                store_out_.write(reinterpret_cast<const char*>(&BinDb::REC_MEDIA), 1);
                store_out_.write(reinterpret_cast<const char*>(&total_body), 8);
                bin_write_str(store_out_, media_id);
                bin_write_str(store_out_, cur_album_id_);
                bin_write_str(store_out_, orig_filename);
                bin_write_str(store_out_, media_kind);
                bin_write_str(store_out_, content_type);
                bin_write_str(store_out_, created_at);
                bin_write_u64(store_out_, data_size);
                if (!store_out_.good()) {
                    fail("write error on store file for media " + media_id);
                    return false;
                }
                media_data_offset_ = static_cast<std::uint64_t>(store_out_.tellp());
            }

            // Stage metadata — do NOT insert into db_.media_ yet.
            // The record becomes visible only after all data is written
            // (in advance_after_media) to prevent concurrent readers from
            // seeing a media entry whose data is still being streamed.
            if (cur_media_new_) {
                MediaRecord m;
                m.id = media_id;
                m.original_filename = orig_filename;
                m.media_kind = media_kind;
                m.content_type = content_type;
                m.created_at = created_at;
                m.size_bytes = data_size;
                m.album_id = cur_album_id_;
                m.store_id = cur_store_id_;
                if (!stream_to_cluster_) {
                    m.record_offset = media_record_offset_;
                    m.data_offset = media_data_offset_;
                }
                pending_media_ = std::move(m);
                ++total_media_;
            }

            if (data_size == 0) { advance_after_media(); return true; }
            phase_ = Phase::MEDIA_DATA;
            return true;
        }

        case Phase::MEDIA_DATA: {
            drain_media_data();
            if (data_written_ >= data_size_) {
                advance_after_media();
                return true;
            }
            return false; // need more data
        }

        default:
            return false;
        }
    }

    void drain_media_data() {
        size_t avail = buf_.size() - pos_;
        size_t remaining = static_cast<size_t>(data_size_ - data_written_);
        size_t n = std::min(avail, remaining);
        if (n == 0) return;

        if (cur_media_new_) {
            if (stream_to_cluster_) {
                // Write media data inline into store buffer
                store_meta_buf_.write(buf_.data() + pos_, static_cast<std::streamsize>(n));
            } else if (store_out_.is_open()) {
                // Write directly to store file
                store_out_.write(buf_.data() + pos_, static_cast<std::streamsize>(n));
                if (!store_out_.good()) {
                    fail("write error on store file for store " + cur_store_id_);
                    return;
                }
            }
        }
        // Skip data for existing media (just advance pos_)
        pos_ += n;
        data_written_ += n;
    }

    void fail(const std::string& msg) {
        {
            std::unique_lock lock(error_mutex_);
            error_msg_ = msg;
        }
        if (store_out_.is_open()) store_out_.close();
        phase_ = Phase::DONE;
        done_.store(true, std::memory_order_release);
    }

    void advance_after_media() {
        // Commit the media record now that all data is on disk.
        if (cur_media_new_ && pending_media_) {
            const std::string mid = pending_media_->id;
            const std::string aid = pending_media_->album_id;
            db_.media_[mid] = std::move(*pending_media_);
            db_.albums_[aid].media_ids.push_back(mid);
            pending_media_.reset();
        }
        ++mi_;
        if (mi_ < num_media_) {
            phase_ = Phase::MEDIA_HEADER;
        } else {
            advance_after_album();
        }
    }

    void advance_after_album() {
        ++ai_;
        if (ai_ < num_albums_) {
            phase_ = Phase::ALBUM_HEADER;
        } else {
            finish_store();
        }
    }

    void finish_store() {
        if (store_out_.is_open()) {
            store_out_.close();
            db_.mmap_store(cur_store_id_);
        }
        if (stream_to_cluster_) {
            // Patch album count and defer replication (outside db_.mutex_)
            std::string buf_str = store_meta_buf_.str();
            std::memcpy(buf_str.data() + 4, &store_album_count_, 4);
            auto data = std::make_shared<std::vector<uint8_t>>(buf_str.begin(), buf_str.end());
            std::string key = "store:" + cur_store_id_;
            {
                std::lock_guard<std::mutex> dlk(deferred_mutex_);
                pending_ops_.push_back([this, key, data]() {
                    on_replicate_(key, data->data(), data->size());
                });
                if (on_store_end_)
                    pending_ops_.push_back(on_store_end_);
            }
        }
        ++si_;
        if (si_ < num_stores_) {
            phase_ = Phase::STORE_HEADER;
        } else {
            finish_import();
        }
    }

    void finish_import() {
        // Vacuum touched stores to compact partial/duplicate records
        // from previous failed imports that used ios::app.
        if (!stream_to_cluster_) {
            for (const auto& sid : touched_stores_)
                db_.vacuum_store_locked(sid);
        }
        db_.save_index_locked();
        if (on_replicate_) {
            auto buf = std::make_shared<std::vector<uint8_t>>(
                db_.save_index_to_buffer_locked());
            if (!buf->empty()) {
                std::lock_guard<std::mutex> dlk(deferred_mutex_);
                pending_ops_.push_back([this, buf]() {
                    on_replicate_("index", buf->data(), buf->size());
                });
            }
        }
        phase_ = Phase::DONE;
        ok_.store(true, std::memory_order_release);
        done_.store(true, std::memory_order_release);
    }

    // ---- members ----

    BinDb& db_;
    ReplicateFn on_replicate_;
    BeginReplicateFn begin_replicate_;
    BinDb::StoreLifecycleFn on_store_begin_;
    BinDb::StoreLifecycleFn on_store_end_;
    bool stream_to_cluster_;

    std::vector<char> buf_;
    size_t pos_ = 0;

    enum class Phase {
        MAGIC, NUM_STORES,
        STORE_HEADER, NUM_ALBUMS,
        ALBUM_HEADER, NUM_MEDIA,
        MEDIA_HEADER, MEDIA_DATA,
        DONE
    } phase_ = Phase::MAGIC;

    bool is_v2_ = false;
    std::uint32_t num_stores_ = 0, si_ = 0;
    std::uint32_t num_albums_ = 0, ai_ = 0;
    std::uint32_t num_media_ = 0, mi_ = 0;

    std::string cur_store_id_, cur_store_name_, cur_store_created_;
    std::string cur_album_id_;
    std::string cur_media_id_;
    bool cur_media_new_ = false;
    std::uint64_t data_size_ = 0, data_written_ = 0;
    std::uint64_t media_record_offset_ = 0, media_data_offset_ = 0;
    std::optional<MediaRecord> pending_media_;

    // Non-cluster: store file handle
    std::ofstream store_out_;

    // Cluster: per-store metadata + data buffer
    std::ostringstream store_meta_buf_{std::ios::binary};
    std::uint32_t store_album_count_ = 0;

    // Deferred cluster operations dispatched outside db_.mutex_
    std::mutex deferred_mutex_;
    std::vector<std::function<void()>> pending_ops_;

    void drain_deferred() {
        auto ops = take_deferred();
        for (auto& op : ops) op();
    }

    std::atomic<bool> done_{false}, ok_{false};
    mutable std::shared_mutex error_mutex_;
    std::string error_msg_;
    size_t total_media_ = 0;
    size_t total_fed_ = 0;
    std::set<std::string> touched_stores_;
};

std::unique_ptr<ImportSession> BinDb::begin_import() {
    return std::make_unique<BinDbImportSession>(*this, nullptr, nullptr);
}

std::unique_ptr<ImportSession> BinDb::begin_import(
    ReplicateFn on_replicate,
    BeginReplicateFn begin_replicate,
    StoreLifecycleFn on_store_begin,
    StoreLifecycleFn on_store_end) {
    return std::make_unique<BinDbImportSession>(*this, std::move(on_replicate),
                                                 std::move(begin_replicate),
                                                 std::move(on_store_begin),
                                                 std::move(on_store_end));
}

// ---- reload / store_ids ----

void BinDb::reload() {
    if (memory_only_) return;  // memory-only mode has no files to reload from
    std::unique_lock lock(mutex_);
    stores_.clear();
    albums_.clear();
    media_.clear();
    mmaps_.clear();
    load_index();
    for (auto& [sid, store] : stores_) {
        load_store(sid, store);
    }
    mmap_all_stores();
}

std::vector<std::string> BinDb::store_ids() const {
    std::shared_lock lock(mutex_);
    std::vector<std::string> ids;
    ids.reserve(stores_.size());
    for (const auto& [k, _] : stores_) ids.push_back(k);
    return ids;
}

// ---- load ----

void BinDb::load() {
    if (memory_only_) return;
    load_index();
    for (auto& [sid, store] : stores_) {
        load_store(sid, store);
    }
    mmap_all_stores();
}

void BinDb::load_index() {
    std::ifstream in(index_file(), std::ios::binary);
    if (!in.is_open()) return;

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v4 = (std::memcmp(magic, INDEX_MAGIC, 4) == 0);
    const bool is_v3 = is_v4 || (std::memcmp(magic, INDEX_MAGIC_V3, 4) == 0);
    if (!is_v3 && std::memcmp(magic, INDEX_MAGIC_V2, 4) != 0) return;

    auto num_stores = bin_read_u32(in);
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        StoreRecord s;
        s.id = bin_read_str(in);
        s.name = bin_read_str(in);
        s.created_at = bin_read_str(in);
        if (is_v3) {
            auto num_domains = bin_read_u32(in);
            for (std::uint32_t di = 0; di < num_domains && in.good(); ++di) {
                AuthDomain d;
                d.id = bin_read_str(in);
                d.label = bin_read_str(in);
                d.url = bin_read_str(in);
                d.client_name = bin_read_str(in);
                d.client_secret = bin_read_str(in);
                s.auth_domains.push_back(std::move(d));
            }
        }
        if (is_v4) {
            auto num_acls = bin_read_u32(in);
            for (std::uint32_t ai = 0; ai < num_acls && in.good(); ++ai) {
                StoreACL acl;
                acl.id = bin_read_str(in);
                acl.group_id = bin_read_str(in);
                acl.group_name = bin_read_str(in);
                std::uint8_t flags = 0;
                in.read(reinterpret_cast<char*>(&flags), 1);
                acl.can_read  = (flags & 0x01) != 0;
                acl.can_write = (flags & 0x02) != 0;
                s.acls.push_back(std::move(acl));
            }
        }
        stores_[s.id] = std::move(s);
    }
}

void BinDb::load_store(const std::string& store_id, StoreRecord& store) {
    std::ifstream in(store_file(store_id), std::ios::binary);
    if (!in.is_open()) return;

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v3 = (std::memcmp(magic, STORE_MAGIC_V3, 4) == 0);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v3 && !is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    if (is_v3) {
        // MDS3: flat record stream with 4 bytes reserved padding after magic
        in.seekg(8);
        while (in.good()) {
            auto rec_off = static_cast<std::uint64_t>(in.tellg());
            std::uint8_t rec_type = 0;
            in.read(reinterpret_cast<char*>(&rec_type), 1);
            if (!in.good()) break;
            std::uint64_t body_size = 0;
            in.read(reinterpret_cast<char*>(&body_size), 8);
            if (!in.good()) break;

            // Next record starts after header (9 bytes) + body
            auto next_rec = static_cast<std::streamoff>(rec_off + 9 + body_size);

            if (rec_type == REC_ALBUM) {
                AlbumRecord a;
                a.id = bin_read_str(in);
                a.name = bin_read_str(in);
                a.created_at = bin_read_str(in);
                { std::uint8_t pub = 0;
                  in.read(reinterpret_cast<char*>(&pub), 1);
                  a.is_public = (pub != 0); }
                a.store_id = store_id;
                a.record_offset = rec_off;

                // Later album records with same ID overwrite earlier ones (for edits)
                auto existing = albums_.find(a.id);
                if (existing != albums_.end()) {
                    a.media_ids = std::move(existing->second.media_ids);
                    albums_[a.id] = std::move(a);
                } else {
                    const std::string aid = a.id;
                    albums_[aid] = std::move(a);
                    store.album_ids.push_back(aid);
                }
            } else if (rec_type == REC_MEDIA) {
                MediaRecord m;
                m.id = bin_read_str(in);
                m.album_id = bin_read_str(in);
                m.original_filename = bin_read_str(in);
                m.media_kind = bin_read_str(in);
                m.content_type = bin_read_str(in);
                m.created_at = bin_read_str(in);
                m.size_bytes = bin_read_u64(in);
                m.store_id = store_id;
                m.record_offset = rec_off;
                m.data_offset = static_cast<std::uint64_t>(in.tellg());
                // Skip raw data — don't read it into memory

                auto ait = albums_.find(m.album_id);
                if (ait != albums_.end()) {
                    auto& mids = ait->second.media_ids;
                    if (std::find(mids.begin(), mids.end(), m.id) == mids.end()) {
                        mids.push_back(m.id);
                    }
                }
                media_[m.id] = std::move(m);
            }
            // Jump to next record (skips REC_EMPTY, unknown types,
            // and any remaining body bytes not consumed above)
            in.seekg(next_rec);
        }
        return;
    }

    // MDS1/MDS2: hierarchical format (legacy)
    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;
            m.data_offset = static_cast<std::uint64_t>(in.tellg());
            in.seekg(static_cast<std::streamoff>(m.size_bytes), std::ios::cur);

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        const std::string aid = a.id;
        store.album_ids.push_back(aid);
        albums_[aid] = std::move(a);
    }
}

// ---- save ----

void BinDb::save_index_locked() const {
    if (memory_only_) return;
    const std::string tmp = index_file().string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;
        out.write(INDEX_MAGIC, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(stores_.size()));
        for (const auto& [sid, s] : stores_) {
            bin_write_str(out, s.id);
            bin_write_str(out, s.name);
            bin_write_str(out, s.created_at);
            bin_write_u32(out, static_cast<std::uint32_t>(s.auth_domains.size()));
            for (const auto& d : s.auth_domains) {
                bin_write_str(out, d.id);
                bin_write_str(out, d.label);
                bin_write_str(out, d.url);
                bin_write_str(out, d.client_name);
                bin_write_str(out, d.client_secret);
            }
            bin_write_u32(out, static_cast<std::uint32_t>(s.acls.size()));
            for (const auto& acl : s.acls) {
                bin_write_str(out, acl.id);
                bin_write_str(out, acl.group_id);
                bin_write_str(out, acl.group_name);
                std::uint8_t flags = 0;
                if (acl.can_read)  flags |= 0x01;
                if (acl.can_write) flags |= 0x02;
                out.write(reinterpret_cast<const char*>(&flags), 1);
            }
        }
    }
    fs::rename(tmp, index_file());
}

void BinDb::save_store_locked(const std::string& store_id) {
    if (memory_only_) return;  // in memory-only mode, data stays in pending_data
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    const auto& store = sit->second;

    const fs::path sf = store_file(store_id);
    std::ifstream old_in(sf, std::ios::binary);

    const std::string tmp = sf.string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;
        out.write(STORE_MAGIC_V2, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(store.album_ids.size()));

        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) continue;
            const auto& a = ait->second;

            bin_write_str(out, a.id);
            bin_write_str(out, a.name);
            bin_write_str(out, a.created_at);
            { std::uint8_t pub = a.is_public ? 1 : 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
            bin_write_u32(out, static_cast<std::uint32_t>(a.media_ids.size()));

            for (const auto& mid : a.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                auto& m = mit->second;

                bin_write_str(out, m.id);
                bin_write_str(out, m.original_filename);
                bin_write_str(out, m.media_kind);
                bin_write_str(out, m.content_type);
                bin_write_str(out, m.created_at);
                bin_write_u64(out, m.size_bytes);

                const auto new_offset = static_cast<std::uint64_t>(out.tellp());

                if (!m.pending_data.empty()) {
                    out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                    m.pending_data.clear();
                    m.pending_data.shrink_to_fit();
                } else if (old_in.is_open() && m.data_offset > 0 && m.size_bytes > 0) {
                    old_in.seekg(static_cast<std::streamoff>(m.data_offset));
                    constexpr std::size_t BUF = 1024 * 1024;
                    std::vector<char> buf(BUF);
                    std::uint64_t remaining = m.size_bytes;
                    while (remaining > 0 && old_in.good()) {
                        auto chunk = static_cast<std::streamsize>(std::min<std::uint64_t>(remaining, BUF));
                        old_in.read(buf.data(), chunk);
                        auto got = old_in.gcount();
                        if (got > 0) out.write(buf.data(), got);
                        remaining -= static_cast<std::uint64_t>(got);
                        if (got < chunk) break;
                    }
                }

                m.data_offset = new_offset;
            }
        }
    }
    old_in.close();
    fs::rename(tmp, sf);
    mmap_store(store_id);
}

// ---- MDS3: append-only store operations ----

void BinDb::ensure_store_file_locked(const std::string& store_id) {
    if (memory_only_) return;
    if (store_id.empty()) return;
    const fs::path sf = store_file(store_id);
    if (fs::exists(sf)) return;
    std::ofstream out(sf, std::ios::binary | std::ios::trunc);
    if (!out.is_open()) return;
    out.write(STORE_MAGIC_V3, 4);
    std::uint32_t reserved = 0;
    out.write(reinterpret_cast<const char*>(&reserved), 4);
}

void BinDb::append_album_locked(const std::string& store_id, const AlbumRecord& album) {
    if (memory_only_) return;
    ensure_store_file_locked(store_id);
    const fs::path sf = store_file(store_id);
    std::ofstream out(sf, std::ios::binary | std::ios::app);
    if (!out.is_open()) return;

    // Build album body
    std::ostringstream body(std::ios::binary);
    bin_write_str(body, album.id);
    bin_write_str(body, album.name);
    bin_write_str(body, album.created_at);
    { std::uint8_t pub = album.is_public ? 1 : 0;
      body.write(reinterpret_cast<const char*>(&pub), 1); }
    std::string body_str = body.str();

    // Record offset is the current end of file
    auto rec_off = static_cast<std::uint64_t>(out.tellp());

    // Write record header: type(u8) + body_size(u64)
    out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
    auto bsz = static_cast<std::uint64_t>(body_str.size());
    out.write(reinterpret_cast<const char*>(&bsz), 8);
    out.write(body_str.data(), static_cast<std::streamsize>(body_str.size()));

    // Update in-memory record offset
    auto ait = albums_.find(album.id);
    if (ait != albums_.end()) ait->second.record_offset = rec_off;

    mmap_store(store_id);
}

void BinDb::append_media_locked(const std::string& store_id, MediaRecord& media) {
    if (memory_only_) return;
    ensure_store_file_locked(store_id);
    const fs::path sf = store_file(store_id);
    std::ofstream out(sf, std::ios::binary | std::ios::app);
    if (!out.is_open()) return;

    // Build media body (metadata portion)
    std::ostringstream meta(std::ios::binary);
    bin_write_str(meta, media.id);
    bin_write_str(meta, media.album_id);
    bin_write_str(meta, media.original_filename);
    bin_write_str(meta, media.media_kind);
    bin_write_str(meta, media.content_type);
    bin_write_str(meta, media.created_at);
    bin_write_u64(meta, media.size_bytes);
    std::string meta_str = meta.str();

    auto body_size = static_cast<std::uint64_t>(meta_str.size()) + media.size_bytes;

    // Record offset
    auto rec_off = static_cast<std::uint64_t>(out.tellp());

    // Write record header: type(u8) + body_size(u64)
    out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
    out.write(reinterpret_cast<const char*>(&body_size), 8);

    // Write metadata
    out.write(meta_str.data(), static_cast<std::streamsize>(meta_str.size()));

    // data_offset = current file position (start of raw data)
    auto data_off = static_cast<std::uint64_t>(out.tellp());

    // Write raw data
    if (!media.pending_data.empty()) {
        out.write(reinterpret_cast<const char*>(media.pending_data.data()),
                  static_cast<std::streamsize>(media.pending_data.size()));
        media.pending_data.clear();
        media.pending_data.shrink_to_fit();
    }

    media.record_offset = rec_off;
    media.data_offset = data_off;

    mmap_store(store_id);
}

void BinDb::zero_record_locked(const std::string& store_id, std::uint64_t record_offset) {
    if (memory_only_) return;
    if (record_offset == 0) return;
    const fs::path sf = store_file(store_id);
    int fd = ::open(sf.c_str(), O_RDWR);
    if (fd < 0) return;

    // Mark as deleted: single byte write (type = REC_EMPTY)
    // body_size stays intact so load/vacuum can skip over the record
    std::uint8_t empty_type = REC_EMPTY;
    (void)::pwrite(fd, &empty_type, 1, static_cast<off_t>(record_offset));

    ::close(fd);
    mmap_store(store_id);
}

void BinDb::vacuum_store_locked(const std::string& store_id) {
    if (memory_only_) return;
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    const auto& store = sit->second;
    const fs::path sf = store_file(store_id);
    if (!fs::exists(sf)) return;

    const std::string tmp = sf.string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;

        // Write MDS3 header
        out.write(STORE_MAGIC_V3, 4);
        std::uint32_t reserved = 0;
        out.write(reinterpret_cast<const char*>(&reserved), 4);

        std::ifstream old_in(sf, std::ios::binary);

        // Write all valid albums
        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) continue;
            auto& a = ait->second;

            std::ostringstream body(std::ios::binary);
            bin_write_str(body, a.id);
            bin_write_str(body, a.name);
            bin_write_str(body, a.created_at);
            { std::uint8_t pub = a.is_public ? 1 : 0;
              body.write(reinterpret_cast<const char*>(&pub), 1); }
            std::string body_str = body.str();

            a.record_offset = static_cast<std::uint64_t>(out.tellp());
            out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
            auto bsz = static_cast<std::uint64_t>(body_str.size());
            out.write(reinterpret_cast<const char*>(&bsz), 8);
            out.write(body_str.data(), static_cast<std::streamsize>(body_str.size()));

            // Write all valid media in this album
            for (const auto& mid : a.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                auto& m = mit->second;

                std::ostringstream mbody(std::ios::binary);
                bin_write_str(mbody, m.id);
                bin_write_str(mbody, m.album_id);
                bin_write_str(mbody, m.original_filename);
                bin_write_str(mbody, m.media_kind);
                bin_write_str(mbody, m.content_type);
                bin_write_str(mbody, m.created_at);
                bin_write_u64(mbody, m.size_bytes);
                std::string mbody_str = mbody.str();

                auto total_body = static_cast<std::uint64_t>(mbody_str.size()) + m.size_bytes;

                m.record_offset = static_cast<std::uint64_t>(out.tellp());
                out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
                out.write(reinterpret_cast<const char*>(&total_body), 8);
                out.write(mbody_str.data(), static_cast<std::streamsize>(mbody_str.size()));

                auto old_data_offset = m.data_offset; // save before vacuum overwrites

                m.data_offset = static_cast<std::uint64_t>(out.tellp());

                // Copy raw data from old file or pending_data
                if (!m.pending_data.empty()) {
                    out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                    m.pending_data.clear();
                    m.pending_data.shrink_to_fit();
                } else if (old_in.is_open() && m.size_bytes > 0 && old_data_offset > 0) {
                    old_in.seekg(static_cast<std::streamoff>(old_data_offset));
                    constexpr std::size_t BUF = 1024 * 1024;
                    std::vector<char> buf(BUF);
                    std::uint64_t rem = m.size_bytes;
                    while (rem > 0 && old_in.good()) {
                        auto chunk = static_cast<std::streamsize>(std::min<std::uint64_t>(rem, BUF));
                        old_in.read(buf.data(), chunk);
                        auto got = old_in.gcount();
                        if (got > 0) out.write(buf.data(), got);
                        rem -= static_cast<std::uint64_t>(got);
                        if (got < chunk) break;
                    }
                }
            }
        }
    }
    fs::rename(tmp, sf);
    mmap_store(store_id);
}

void BinDb::vacuum_store(const std::string& store_id) {
    std::unique_lock lock(mutex_);
    vacuum_store_locked(store_id);
}

void BinDb::vacuum_all_stores() {
    std::unique_lock lock(mutex_);
    for (const auto& [sid, _] : stores_) {
        vacuum_store_locked(sid);
    }
}

// ---- mmap management ----

void BinDb::mmap_store(const std::string& store_id) {
    auto path = store_file(store_id);
    if (fs::exists(path) && fs::file_size(path) > 0) {
        mmaps_[store_id] = MmapRegion::open(path);
    }
}

void BinDb::mmap_all_stores() {
    for (const auto& [sid, _] : stores_) {
        mmap_store(sid);
    }
}

// ---- buffer-based save/load (for cluster-only storage) ----

std::vector<std::uint8_t> BinDb::save_index_to_buffer() const {
    std::shared_lock lock(mutex_);
    return save_index_to_buffer_locked();
}

std::vector<std::uint8_t> BinDb::save_index_to_buffer_locked() const {
    std::ostringstream oss(std::ios::binary);
    oss.write(INDEX_MAGIC, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(stores_.size()));
    for (const auto& [sid, s] : stores_) {
        bin_write_str(oss, s.id);
        bin_write_str(oss, s.name);
        bin_write_str(oss, s.created_at);
        bin_write_u32(oss, static_cast<std::uint32_t>(s.auth_domains.size()));
        for (const auto& d : s.auth_domains) {
            bin_write_str(oss, d.id);
            bin_write_str(oss, d.label);
            bin_write_str(oss, d.url);
            bin_write_str(oss, d.client_name);
            bin_write_str(oss, d.client_secret);
        }
        bin_write_u32(oss, static_cast<std::uint32_t>(s.acls.size()));
        for (const auto& acl : s.acls) {
            bin_write_str(oss, acl.id);
            bin_write_str(oss, acl.group_id);
            bin_write_str(oss, acl.group_name);
            std::uint8_t flags = 0;
            if (acl.can_read)  flags |= 0x01;
            if (acl.can_write) flags |= 0x02;
            oss.write(reinterpret_cast<const char*>(&flags), 1);
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_index_from_buffer(const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v4 = (std::memcmp(magic, INDEX_MAGIC, 4) == 0);
    const bool is_v3 = is_v4 || (std::memcmp(magic, INDEX_MAGIC_V3, 4) == 0);
    if (!is_v3 && std::memcmp(magic, INDEX_MAGIC_V2, 4) != 0) return;

    // Preserve existing store IDs to detect removed stores
    std::set<std::string> old_ids;
    for (const auto& [k, _] : stores_) old_ids.insert(k);

    auto num_stores = bin_read_u32(in);
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        StoreRecord s;
        s.id = bin_read_str(in);
        s.name = bin_read_str(in);
        s.created_at = bin_read_str(in);
        if (is_v3) {
            auto num_domains = bin_read_u32(in);
            for (std::uint32_t di = 0; di < num_domains && in.good(); ++di) {
                AuthDomain d;
                d.id = bin_read_str(in);
                d.label = bin_read_str(in);
                d.url = bin_read_str(in);
                d.client_name = bin_read_str(in);
                d.client_secret = bin_read_str(in);
                s.auth_domains.push_back(std::move(d));
            }
        }
        if (is_v4) {
            auto num_acls = bin_read_u32(in);
            for (std::uint32_t ai = 0; ai < num_acls && in.good(); ++ai) {
                StoreACL acl;
                acl.id = bin_read_str(in);
                acl.group_id = bin_read_str(in);
                acl.group_name = bin_read_str(in);
                std::uint8_t flags = 0;
                in.read(reinterpret_cast<char*>(&flags), 1);
                acl.can_read  = (flags & 0x01) != 0;
                acl.can_write = (flags & 0x02) != 0;
                s.acls.push_back(std::move(acl));
            }
        }
        // Keep existing album_ids if store already present
        auto existing = stores_.find(s.id);
        if (existing != stores_.end()) {
            s.album_ids = existing->second.album_ids;
        }
        stores_[s.id] = std::move(s);
        old_ids.erase(s.id);
    }

    // Do not remove locally-present stores that are absent from the buffer.
    // Deletes propagate explicitly via delete_store() + cluster_.remove();
    // removing here would wipe out recently-created local stores when the
    // synced index is stale (same merge-only approach as authdb).
}

std::vector<std::uint8_t> BinDb::save_store_to_buffer(const std::string& store_id) {
    std::shared_lock lock(mutex_);
    return save_store_to_buffer_locked(store_id);
}

std::vector<std::uint8_t> BinDb::save_store_to_buffer_locked(const std::string& store_id) const {
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return {};
    const auto& store = sit->second;

    std::ostringstream oss(std::ios::binary);
    oss.write(STORE_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(store.album_ids.size()));

    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait == albums_.end()) continue;
        const auto& a = ait->second;

        bin_write_str(oss, a.id);
        bin_write_str(oss, a.name);
        bin_write_str(oss, a.created_at);
        { std::uint8_t pub = a.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
        bin_write_u32(oss, static_cast<std::uint32_t>(a.media_ids.size()));

        for (const auto& mid : a.media_ids) {
            auto mit = media_.find(mid);
            if (mit == media_.end()) continue;
            const auto& m = mit->second;

            bin_write_str(oss, m.id);
            bin_write_str(oss, m.original_filename);
            bin_write_str(oss, m.media_kind);
            bin_write_str(oss, m.content_type);
            bin_write_str(oss, m.created_at);
            bin_write_u64(oss, m.size_bytes);

            // In memory-only mode, data is always in pending_data
            if (!m.pending_data.empty()) {
                oss.write(reinterpret_cast<const char*>(m.pending_data.data()),
                          static_cast<std::streamsize>(m.pending_data.size()));
            }
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_store_from_buffer(const std::string& store_id, const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    auto& store = sit->second;

    // Clear old albums/media for this store
    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) {
            for (const auto& mid : ait->second.media_ids) media_.erase(mid);
            albums_.erase(ait);
        }
    }
    store.album_ids.clear();

    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;

            // Read media binary data into pending_data (memory-only)
            if (m.size_bytes > 0) {
                m.pending_data.resize(m.size_bytes);
                in.read(reinterpret_cast<char*>(m.pending_data.data()),
                        static_cast<std::streamsize>(m.size_bytes));
            }
            m.data_offset = 0;

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        const std::string aid = a.id;
        store.album_ids.push_back(aid);
        albums_[aid] = std::move(a);
    }
}

// ==================== ClusterMediaBackend ====================

// ---- metadata-only store buffer (no raw media data) ----

std::vector<std::uint8_t> BinDb::save_store_metadata_to_buffer(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    return save_store_metadata_to_buffer_locked(store_id);
}

std::vector<std::uint8_t> BinDb::save_store_metadata_to_buffer_locked(const std::string& store_id) const {
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return {};
    const auto& store = sit->second;

    std::ostringstream oss(std::ios::binary);
    oss.write(STORE_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(store.album_ids.size()));

    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait == albums_.end()) continue;
        const auto& a = ait->second;

        bin_write_str(oss, a.id);
        bin_write_str(oss, a.name);
        bin_write_str(oss, a.created_at);
        { std::uint8_t pub = a.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
        bin_write_u32(oss, static_cast<std::uint32_t>(a.media_ids.size()));

        for (const auto& mid : a.media_ids) {
            auto mit = media_.find(mid);
            if (mit == media_.end()) continue;
            const auto& m = mit->second;

            bin_write_str(oss, m.id);
            bin_write_str(oss, m.original_filename);
            bin_write_str(oss, m.media_kind);
            bin_write_str(oss, m.content_type);
            bin_write_str(oss, m.created_at);
            bin_write_u64(oss, m.size_bytes);
            // NO raw data written — metadata only
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_store_metadata_from_buffer(const std::string& store_id, const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    auto& store = sit->second;

    // Clear old albums/media for this store
    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) {
            for (const auto& mid : ait->second.media_ids) media_.erase(mid);
            albums_.erase(ait);
        }
    }
    store.album_ids.clear();

    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;
            // Skip past inline media data — do NOT load into RAM
            if (m.size_bytes > 0) {
                in.seekg(static_cast<std::streamoff>(m.size_bytes), std::ios::cur);
            }
            m.data_offset = 0;

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        const std::string aid = a.id;
        store.album_ids.push_back(aid);
        albums_[aid] = std::move(a);
    }
}

std::optional<MediaRecord> BinDb::add_media_metadata(const std::string& album_id,
                                                      const std::string& filename,
                                                      const std::string& content_type,
                                                      std::uint64_t size_bytes) {
    std::unique_lock lock(mutex_);
    auto ait = albums_.find(album_id);
    if (ait == albums_.end()) return std::nullopt;

    const fs::path p(filename);
    MediaRecord m;
    m.id = make_uuid();
    m.original_filename = filename;
    m.media_kind = detect_kind(p.extension().string());
    m.content_type = content_type;
    m.album_id = album_id;
    m.store_id = ait->second.store_id;
    m.created_at = now_iso8601();
    m.size_bytes = size_bytes;
    m.data_offset = 0;
    // no pending_data — raw data goes directly to cluster

    const std::string mid = m.id;
    media_[mid] = std::move(m);
    ait->second.media_ids.push_back(mid);

    auto mit = media_.find(mid);
    if (mit == media_.end()) return std::nullopt;
    return mit->second;
}

ClusterMediaBackend::ClusterMediaBackend(Cluster& cluster)
    : local_(), cluster_(cluster) {}

void ClusterMediaBackend::start_sync() {
    sync_from_cluster();
}

ClusterMediaBackend::~ClusterMediaBackend() {
}

// ---- stores ----
StoreRecord ClusterMediaBackend::create_store(const std::string& name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto s = local_.create_store(name);
    replicate_store(s.id);
    replicate_index();
    return s;
}

std::optional<StoreRecord> ClusterMediaBackend::get_store(const std::string& id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_store(id);
}

std::vector<StoreRecord> ClusterMediaBackend::list_stores() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_stores();
}

bool ClusterMediaBackend::delete_store(const std::string& id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.delete_store(id)) return false;
    // Remove store blob (contains all media data inline)
    cluster_.remove("store:" + id);
    replicate_index();
    return true;
}

bool ClusterMediaBackend::rename_store(const std::string& id, const std::string& new_name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.rename_store(id, new_name)) return false;
    replicate_index();
    return true;
}

// ---- store auth domains ----
std::string ClusterMediaBackend::add_store_domain(const std::string& store_id, AuthDomain domain) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto did = local_.add_store_domain(store_id, std::move(domain));
    if (!did.empty()) replicate_index();
    return did;
}

bool ClusterMediaBackend::remove_store_domain(const std::string& store_id, const std::string& domain_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.remove_store_domain(store_id, domain_id)) return false;
    replicate_index();
    return true;
}

std::vector<AuthDomain> ClusterMediaBackend::list_store_domains(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_store_domains(store_id);
}

std::optional<AuthDomain> ClusterMediaBackend::find_domain_by_label(const std::string& label) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.find_domain_by_label(label);
}

// ---- store ACLs ----
std::string ClusterMediaBackend::add_store_acl(const std::string& store_id, StoreACL acl) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto aid = local_.add_store_acl(store_id, std::move(acl));
    if (!aid.empty()) replicate_index();
    return aid;
}

bool ClusterMediaBackend::remove_store_acl(const std::string& store_id, const std::string& acl_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.remove_store_acl(store_id, acl_id)) return false;
    replicate_index();
    return true;
}

std::vector<StoreACL> ClusterMediaBackend::list_store_acls(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_store_acls(store_id);
}

std::vector<std::string> ClusterMediaBackend::all_acl_group_ids() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.all_acl_group_ids();
}

// ---- albums ----
std::optional<AlbumRecord> ClusterMediaBackend::create_album(const std::string& store_id, const std::string& name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto a = local_.create_album(store_id, name);
    if (a) replicate_store(store_id);
    return a;
}

std::optional<AlbumRecord> ClusterMediaBackend::get_album(const std::string& id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_album(id);
}

std::vector<AlbumRecord> ClusterMediaBackend::list_albums(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_albums(store_id);
}

bool ClusterMediaBackend::delete_album(const std::string& id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto album = local_.get_album(id);
    if (!album) return false;
    std::string store_id = album->store_id;
    if (!local_.delete_album(id)) return false;
    // Re-replicate store (album and its media removed from inline blob)
    replicate_store(store_id);
    replicate_index();
    return true;
}

bool ClusterMediaBackend::set_album_public(const std::string& album_id, bool is_public) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto album = local_.get_album(album_id);
    if (!album) return false;
    if (!local_.set_album_public(album_id, is_public)) return false;
    replicate_store(album->store_id);
    return true;
}

bool ClusterMediaBackend::is_album_public(const std::string& album_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.is_album_public(album_id);
}

bool ClusterMediaBackend::is_media_public(const std::string& media_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.is_media_public(media_id);
}

std::vector<AlbumRecord> ClusterMediaBackend::list_public_albums() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_public_albums();
}

// ---- media ----
std::optional<MediaRecord> ClusterMediaBackend::add_media(const std::string& album_id,
                                                            const std::string& filename,
                                                            const std::string& content_type,
                                                            std::vector<std::uint8_t> data) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    // Store raw data separately, keep only metadata in local BinDb
    std::vector<std::uint8_t> raw_data = data;
    auto m = local_.add_media(album_id, filename, content_type, std::move(data));
    if (m) {
        // Replicate media data as separate key
        if (!raw_data.empty()) {
            cluster_.replicate("media:" + m->id, raw_data.data(), raw_data.size());
        }
        // Replicate store metadata (no inline media data)
        replicate_store(m->store_id);
    }
    return m;
}

std::optional<MediaRecord> ClusterMediaBackend::get_media(const std::string& id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_media(id);
}

std::vector<MediaRecord> ClusterMediaBackend::list_media(const std::string& album_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_media(album_id);
}

bool ClusterMediaBackend::delete_media(const std::string& id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto media = local_.get_media(id);
    if (!media) return false;
    std::string store_id = media->store_id;
    if (!local_.delete_media(id)) return false;
    // Remove media data from cluster
    cluster_.remove("media:" + id);
    // Re-replicate store metadata
    replicate_store(store_id);
    replicate_index();
    return true;
}

// ---- raw data access (per-media cluster keys) ----

std::vector<std::uint8_t> ClusterMediaBackend::read_media_data(const std::string& media_id) const {
    // Try local pending_data first (from add_media before replicate)
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto result = local_.read_media_data(media_id);
        if (!result.empty()) return result;
        auto m = local_.get_media(media_id);
        if (!m || m->size_bytes == 0) return {};
    }
    // Fetch media data directly from cluster (per-media key)
    std::vector<uint8_t> out;
    const_cast<Cluster&>(cluster_).fetch("media:" + media_id, out);
    return out;
}

std::vector<std::uint8_t> ClusterMediaBackend::read_media_data_range(const std::string& media_id,
                                                                       std::uint64_t offset,
                                                                       std::uint64_t length) const {
    // Try local pending_data first
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto result = local_.read_media_data_range(media_id, offset, length);
        if (!result.empty()) return result;
        auto m = local_.get_media(media_id);
        if (!m || m->size_bytes == 0) return {};
    }
    // Fetch only needed stripes from cluster (partial retrieve)
    std::vector<uint8_t> out;
    const_cast<Cluster&>(cluster_).fetch_range("media:" + media_id, offset, length, out);
    return out;
}

std::uint64_t ClusterMediaBackend::get_media_size(const std::string& media_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_media_size(media_id);
}

// ---- export / import ----
bool ClusterMediaBackend::export_db(const std::string& path) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    // Export requires raw data — fetch each media from cluster
    auto buf = export_db_to_buffer_impl();
    if (buf.empty()) return false;
    std::ofstream out(path, std::ios::binary | std::ios::trunc);
    if (!out.is_open()) return false;
    out.write(reinterpret_cast<const char*>(buf.data()),
              static_cast<std::streamsize>(buf.size()));
    return out.good();
}

std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return export_db_to_buffer_impl();
}

std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer_impl() const {
    // Build export buffer with raw data fetched from cluster
    static constexpr char EXPORT_MAGIC_V2[4] = {'M', 'D', 'E', '2'};
    std::ostringstream oss(std::ios::binary);
    oss.write(EXPORT_MAGIC_V2, 4);

    auto stores = local_.list_stores();
    bin_write_u32(oss, static_cast<std::uint32_t>(stores.size()));

    for (const auto& store : stores) {
        bin_write_str(oss, store.id);
        bin_write_str(oss, store.name);
        bin_write_str(oss, store.created_at);

        auto albums = local_.list_albums(store.id);
        bin_write_u32(oss, static_cast<std::uint32_t>(albums.size()));

        for (const auto& album : albums) {
            bin_write_str(oss, album.id);
            bin_write_str(oss, album.name);
            bin_write_str(oss, album.created_at);
            { std::uint8_t pub = album.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }

            auto media_list = local_.list_media(album.id);
            bin_write_u32(oss, static_cast<std::uint32_t>(media_list.size()));

            for (const auto& m : media_list) {
                bin_write_str(oss, m.id);
                bin_write_str(oss, m.original_filename);
                bin_write_str(oss, m.media_kind);
                bin_write_str(oss, m.content_type);
                bin_write_str(oss, m.created_at);
                bin_write_u64(oss, m.size_bytes);

                // Fetch raw data from cluster (per-media key)
                std::vector<uint8_t> raw;
                const_cast<Cluster&>(cluster_).fetch("media:" + m.id, raw);
                if (m.size_bytes > 0 && !raw.empty()) {
                    oss.write(reinterpret_cast<const char*>(raw.data()),
                              static_cast<std::streamsize>(raw.size()));
                } else if (m.size_bytes > 0) {
                    // Pad with zeros if fetch failed to maintain format integrity
                    std::vector<uint8_t> zeros(m.size_bytes, 0);
                    oss.write(reinterpret_cast<const char*>(zeros.data()),
                              static_cast<std::streamsize>(zeros.size()));
                }
            }
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

bool ClusterMediaBackend::import_db(const std::string& path) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.import_db(path)) return false;
    // Replicate each store (with inline media data) to cluster
    for (const auto& sid : local_.store_ids()) {
        replicate_store(sid);
    }
    replicate_index();
    return true;
}

bool ClusterMediaBackend::import_db_from_buffer(const std::vector<std::uint8_t>& data) {
    return import_db_from_buffer(data.data(), data.size());
}

bool ClusterMediaBackend::import_db_from_buffer(const std::uint8_t* data, std::size_t len) {
    DBG_LOG("[CLUSTER-IMPORT] start, input_size=" << len << "\n");
    if (!cluster_.isRunning()) {
        DBG_LOG("[CLUSTER-IMPORT] cluster not running!\n");
        return false;
    }

    // Hold cluster_op_mutex_ for the entire import to prevent
    // sync_from_cluster from overwriting newly imported data with stale
    // cluster state.  Use collect_fn so BinDb::mutex_ is NOT held during
    // network I/O (import_from_stream releases it before we replicate).
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    DBG_LOG("[CLUSTER-IMPORT] acquired cluster_op_mutex_\n");

    // Phase 1: Parse and update local state (BinDb::mutex_ held internally, fast).
    // Collect replication buffers instead of calling cluster_.replicate() under
    // BinDb::mutex_ — that would block all reads for the entire network duration.
    std::vector<std::pair<std::string, std::vector<uint8_t>>> repl_queue;
    auto collect_fn = [&repl_queue](const std::string& key, const uint8_t* d, size_t l) {
        DBG_LOG("[CLUSTER-IMPORT] collect key=" << key << " size=" << l << "\n");
        repl_queue.emplace_back(key, std::vector<uint8_t>(d, d + l));
    };
    DBG_LOG("[CLUSTER-IMPORT] Phase 1: parsing...\n");
    bool parse_ok = local_.import_db_from_buffer(data, len, collect_fn);
    DBG_LOG("[CLUSTER-IMPORT] Phase 1 done, ok=" << parse_ok
              << " queued=" << repl_queue.size() << "\n");
    if (!parse_ok) return false;

    // Phase 2: Replicate collected buffers.  BinDb::mutex_ is released
    // (import_from_stream returned), so reads to local_ are not blocked.
    // cluster_op_mutex_ is still held so ClusterMediaBackend reads wait,
    // but sync cannot interfere.
    DBG_LOG("[CLUSTER-IMPORT] Phase 2: replicating " << repl_queue.size() << " buffers\n");
    std::vector<std::pair<std::string, std::vector<uint8_t>>> failed_entries;
    int repl_idx = 0;
    for (auto& [key, buf] : repl_queue) {
        ++repl_idx;
        DBG_LOG("[CLUSTER-IMPORT]   replicate " << repl_idx << "/" << repl_queue.size()
                  << " key=" << key << " size=" << buf.size() << "\n");
        if (!cluster_.replicate(key, buf.data(), buf.size())) {
            DBG_LOG("[CLUSTER-IMPORT]   FAILED key=" << key << "\n");
            failed_entries.emplace_back(std::move(key), std::move(buf));
        } else {
            DBG_LOG("[CLUSTER-IMPORT]   OK key=" << key << "\n");
            buf.clear();
            buf.shrink_to_fit();
        }
    }
    repl_queue.clear();

    // Phase 3: Retry failures (using the original buffers kept from Phase 2)
    if (!failed_entries.empty()) {
        DBG_LOG("[CLUSTER-IMPORT] Phase 3: retrying " << failed_entries.size() << " failed keys\n");
        constexpr int MAX_RETRIES = 3;
        for (int attempt = 1; attempt <= MAX_RETRIES && !failed_entries.empty(); ++attempt) {
            DBG_LOG("[CLUSTER-IMPORT]   retry attempt " << attempt << "\n");
            std::this_thread::sleep_for(std::chrono::seconds(attempt * 2));
            std::vector<std::pair<std::string, std::vector<uint8_t>>> still_failed;
            for (auto& [key, buf] : failed_entries) {
                bool ok = false;
                if (key == "index") {
                    // Re-generate index buffer (cheap)
                    auto idx_buf = local_.save_index_to_buffer();
                    ok = cluster_.replicate("index", idx_buf.data(), idx_buf.size());
                } else {
                    // store: and media: keys — use the original buffer
                    ok = cluster_.replicate(key, buf.data(), buf.size());
                }
                if (!ok) {
                    still_failed.emplace_back(std::move(key), std::move(buf));
                }
            }
            failed_entries = std::move(still_failed);
            if (failed_entries.empty()) {
                break;
            }
        }
    }

    DBG_LOG("[CLUSTER-IMPORT] complete, success\n");
    return true;
}

std::unique_ptr<ImportSession> ClusterMediaBackend::begin_import() {
    if (!cluster_.isRunning()) {
        return nullptr;
    }

    auto replicate_fn = [this](const std::string& key, const uint8_t* d, size_t len) {
        cluster_.replicate(key, d, len);
    };
    return local_.begin_import(std::move(replicate_fn), nullptr,
                               nullptr, nullptr);
}

void ClusterMediaBackend::vacuum_store(const std::string& store_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    local_.vacuum_store(store_id);
    replicate_store(store_id);
    replicate_index();
    cluster_.vacuum_all_nodes();
}

void ClusterMediaBackend::vacuum_all_stores() {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    local_.vacuum_all_stores();
    for (const auto& sid : local_.store_ids()) {
        replicate_store(sid);
    }
    replicate_index();
    cluster_.vacuum_all_nodes();
}

// ---- cluster sync ----
void ClusterMediaBackend::sync_from_cluster() {
    if (!cluster_.isRunning()) return;

    // Fetch index from cluster (contains store/album/ACL metadata, no media data)
    std::vector<uint8_t> index_data;
    cluster_.fetch("index", index_data);
    if (!index_data.empty()) {
        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        local_.load_index_from_buffer(index_data);
    }

    // Snapshot store IDs under shared lock
    std::vector<std::string> sids;
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        sids = local_.store_ids();
    }

    // Fetch each store metadata (album/media records) from cluster.
    // Uses load_store_metadata_from_buffer which loads media metadata
    // but NOT raw media data — keeps RAM usage low.
    for (const auto& sid : sids) {
        std::vector<uint8_t> store_data;
        cluster_.fetch("store:" + sid, store_data);
        if (!store_data.empty()) {
            std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
            local_.load_store_metadata_from_buffer(sid, store_data);
        }
    }
}

BinDb& ClusterMediaBackend::local() { return local_; }

void ClusterMediaBackend::replicate_index() {
    if (!cluster_.isRunning()) return;
    auto buf = local_.save_index_to_buffer();
    if (buf.empty()) {
        return;
    }
    cluster_.replicate("index", buf.data(), buf.size());
}

void ClusterMediaBackend::replicate_store(const std::string& store_id) {
    if (!cluster_.isRunning()) return;
    // Replicate store metadata only (media data stored as separate keys)
    auto buf = local_.save_store_metadata_to_buffer(store_id);
    if (buf.empty()) {
        return;
    }
    cluster_.replicate("store:" + store_id, buf.data(), buf.size());
}

void ClusterMediaBackend::repair_replication() {
    if (!cluster_.isRunning()) return;
    size_t total_peers = cluster_.getTotalPeers();
    if (total_peers < 2) return;

    const auto& cfg = cluster_.getConfig();
    size_t required_shards = cfg.data_blocks + cfg.parity_blocks;

    auto peer_groups = cluster_.list_peer_groups();
    std::vector<std::unordered_set<uint64_t>> node_group_sets(peer_groups.size());
    for (size_t i = 0; i < peer_groups.size(); ++i) {
        node_group_sets[i].insert(peer_groups[i].groups.begin(),
                                  peer_groups[i].groups.end());
    }

    size_t online_count = 0;
    for (const auto& pg : peer_groups)
        if (pg.online) ++online_count;
    if (online_count < required_shards) return; // not enough nodes online to repair

    // Check index
    uint64_t index_gid = cluster_group_id("index");
    int index_shards = 0;
    for (size_t i = 0; i < peer_groups.size(); ++i) {
        if (peer_groups[i].online && node_group_sets[i].count(index_gid))
            ++index_shards;
    }
    if (index_shards < static_cast<int>(required_shards)) {
        replicate_index();
    }

    // Check stores (media data is now included in store blobs)
    for (const auto& sid : local_.store_ids()) {
        uint64_t gid = cluster_group_id("store:" + sid);
        int shards = 0;
        for (size_t i = 0; i < peer_groups.size(); ++i) {
            if (peer_groups[i].online && node_group_sets[i].count(gid))
                ++shards;
        }
        if (shards < static_cast<int>(required_shards)) {
            replicate_store(sid);
        }
    }
}

void ClusterMediaBackend::sync_loop() {
    // No-op: sync thread removed. All cluster operations are synchronous.
}

} // namespace mediadb
