#include "backend.h"
#include "cluster.h"

#include <algorithm>
#include <atomic>
#include <cstdlib>
#include <future>
#include <iostream>
#include <mutex>
#include <sstream>
#include <unordered_map>
#include <unordered_set>

#ifdef NDEBUG
#define DBG_LOG(x) do {} while(0)
#else
#define DBG_LOG(x) do { std::cerr << x; } while(0)
#endif

namespace mediadb {

static int sync_interval_seconds() {
    const char* v = std::getenv("MEDIADB_SYNC_INTERVAL_SEC");
    if (!v || v[0] == '\0') return 60;
    int x = std::atoi(v);
    if (x < 1) x = 1;
    if (x > 3600) x = 3600;
    return x;
}

// Zero-copy read-only streambuf over a contiguous byte buffer
class MemBuf : public std::streambuf {
public:
    MemBuf(const std::uint8_t* data, std::size_t size) {
        auto* p = const_cast<char*>(reinterpret_cast<const char*>(data));
        setg(p, p, p + size);
    }

protected:
    pos_type seekoff(off_type off, std::ios_base::seekdir dir,
                     std::ios_base::openmode which = std::ios_base::in) override {
        if (which & std::ios_base::in) {
            char* p = gptr();
            if (dir == std::ios_base::beg) p = eback() + off;
            else if (dir == std::ios_base::cur) p += off;
            else if (dir == std::ios_base::end) p = egptr() + off;
            if (p < eback() || p > egptr()) return pos_type(off_type(-1));
            setg(eback(), p, egptr());
            return pos_type(p - eback());
        }
        return pos_type(off_type(-1));
    }

    pos_type seekpos(pos_type pos,
                     std::ios_base::openmode which = std::ios_base::in) override {
        return seekoff(off_type(pos), std::ios_base::beg, which);
    }
};

// ---- BinDb ----

BinDb::BinDb(const std::string& db_dir) : db_dir_(db_dir), memory_only_(false) {
    fs::create_directories(db_dir_);
    load();
}

BinDb::BinDb() : memory_only_(true) {}

fs::path BinDb::index_file() const { return fs::path(db_dir_) / "index.mdb"; }
fs::path BinDb::store_file(const std::string& store_id) const { return fs::path(db_dir_) / (store_id + ".mdb"); }

std::string BinDb::detect_kind(std::string ext) {
    std::transform(ext.begin(), ext.end(), ext.begin(),
                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
    if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" || ext == ".bmp" || ext == ".gif" || ext == ".avif") return "image";
    if (ext == ".mp4" || ext == ".mov" || ext == ".mkv" || ext == ".webm" || ext == ".avi" || ext == ".m4v" || ext == ".ts" || ext == ".flv") return "video";
    if (ext == ".mp3" || ext == ".flac" || ext == ".wav" || ext == ".ogg" || ext == ".aac" || ext == ".m4a" || ext == ".opus") return "audio";
    return "unknown";
}

// ---- stores ----

StoreRecord BinDb::create_store(const std::string& name) {
    StoreRecord s;
    {
        std::unique_lock lock(mutex_);
        s.id = make_uuid();
        s.name = name;
        s.created_at = now_iso8601();
        stores_[s.id] = s;
        save_index_locked();
        ensure_store_file_locked(s.id);
    }
    return s;
}

std::optional<StoreRecord> BinDb::get_store(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(id);
    if (it == stores_.end()) return std::nullopt;
    return it->second;
}

std::vector<StoreRecord> BinDb::list_stores() const {
    std::shared_lock lock(mutex_);
    std::vector<StoreRecord> out;
    out.reserve(stores_.size());
    for (const auto& [k, v] : stores_) out.push_back(v);
    return out;
}

bool BinDb::delete_store(const std::string& id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(id);
        if (it == stores_.end()) return false;
        for (const auto& d : it->second.auth_domains) {
            domain_label_index_.erase(d.label);
        }
        for (const auto& aid : it->second.album_ids) {
            auto ait = albums_.find(aid);
            if (ait != albums_.end()) {
                for (const auto& mid : ait->second.media_ids) media_.erase(mid);
                albums_.erase(ait);
            }
        }
        mmaps_.erase(id);
        if (!memory_only_) {
            auto sf = store_file(id);
            if (fs::exists(sf)) fs::remove(sf);
        }
        stores_.erase(it);
        if (!memory_only_) save_index_locked();
    }
    return true;
}

bool BinDb::rename_store(const std::string& id, const std::string& new_name) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(id);
        if (it == stores_.end()) return false;
        it->second.name = new_name;
        save_index_locked();
    }
    return true;
}

// ---- store auth domains ----

std::string BinDb::add_store_domain(const std::string& store_id, AuthDomain domain) {
    std::string did;
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return {};
        domain.id = make_uuid();
        did = domain.id;
        domain_label_index_[domain.label] = {store_id, domain.id};
        it->second.auth_domains.push_back(std::move(domain));
        save_index_locked();
    }
    return did;
}

bool BinDb::remove_store_domain(const std::string& store_id, const std::string& domain_id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return false;
        auto& domains = it->second.auth_domains;
        auto dit = std::find_if(domains.begin(), domains.end(),
                                [&](const AuthDomain& d) { return d.id == domain_id; });
        if (dit == domains.end()) return false;
        domain_label_index_.erase(dit->label);
        domains.erase(dit);
        save_index_locked();
    }
    return true;
}

std::vector<AuthDomain> BinDb::list_store_domains(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(store_id);
    if (it == stores_.end()) return {};
    return it->second.auth_domains;
}

std::optional<AuthDomain> BinDb::find_domain_by_label(const std::string& label) const {
    std::shared_lock lock(mutex_);
    auto it = domain_label_index_.find(label);
    if (it == domain_label_index_.end()) return std::nullopt;
    auto sit = stores_.find(it->second.first);
    if (sit == stores_.end()) return std::nullopt;
    for (const auto& d : sit->second.auth_domains) {
        if (d.id == it->second.second) return d;
    }
    return std::nullopt;
}

// ---- store ACLs ----

std::string BinDb::add_store_acl(const std::string& store_id, StoreACL acl) {
    std::string aid;
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return {};
        acl.id = make_uuid();
        aid = acl.id;
        it->second.acls.push_back(std::move(acl));
        save_index_locked();
    }
    return aid;
}

bool BinDb::remove_store_acl(const std::string& store_id, const std::string& acl_id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return false;
        auto& acls = it->second.acls;
        auto ait = std::find_if(acls.begin(), acls.end(),
                                [&](const StoreACL& a) { return a.id == acl_id; });
        if (ait == acls.end()) return false;
        acls.erase(ait);
        save_index_locked();
    }
    return true;
}

std::vector<StoreACL> BinDb::list_store_acls(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(store_id);
    if (it == stores_.end()) return {};
    return it->second.acls;
}

std::vector<std::string> BinDb::all_acl_group_ids() const {
    std::shared_lock lock(mutex_);
    std::set<std::string> ids;
    for (const auto& [sid, s] : stores_) {
        for (const auto& acl : s.acls) {
            ids.insert(acl.group_id);
        }
    }
    return {ids.begin(), ids.end()};
}

// ---- albums ----

std::optional<AlbumRecord> BinDb::create_album(const std::string& store_id, const std::string& name) {
    std::optional<AlbumRecord> result;
    {
        std::unique_lock lock(mutex_);
        auto sit = stores_.find(store_id);
        if (sit == stores_.end()) return std::nullopt;

        AlbumRecord a;
        a.id = make_uuid();
        a.name = name;
        a.store_id = store_id;
        a.created_at = now_iso8601();
        albums_[a.id] = a;
        sit->second.album_ids.push_back(a.id);
        append_album_locked(store_id, albums_[a.id]);
        result = albums_[a.id];
    }
    return result;
}

std::optional<AlbumRecord> BinDb::get_album(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = albums_.find(id);
    if (it == albums_.end()) return std::nullopt;
    return it->second;
}

std::vector<AlbumRecord> BinDb::list_albums(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    std::vector<AlbumRecord> out;
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return out;
    out.reserve(sit->second.album_ids.size());
    for (const auto& aid : sit->second.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) out.push_back(ait->second);
    }
    return out;
}

bool BinDb::delete_album(const std::string& id) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto ait = albums_.find(id);
        if (ait == albums_.end()) return false;
        store_id = ait->second.store_id;
        auto sit = stores_.find(store_id);
        if (sit != stores_.end()) {
            auto& aids = sit->second.album_ids;
            aids.erase(std::remove(aids.begin(), aids.end(), id), aids.end());
        }
        // Zero all media records belonging to this album
        for (const auto& mid : ait->second.media_ids) {
            auto mit2 = media_.find(mid);
            if (mit2 != media_.end()) {
                zero_record_locked(store_id, mit2->second.record_offset);
                media_.erase(mit2);
            }
        }
        // Zero the album record itself
        zero_record_locked(store_id, ait->second.record_offset);
        albums_.erase(ait);
    }
    return true;
}

bool BinDb::set_album_public(const std::string& album_id, bool is_public) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto it = albums_.find(album_id);
        if (it == albums_.end()) return false;
        // Zero old album record, update flag, append new one
        zero_record_locked(it->second.store_id, it->second.record_offset);
        it->second.is_public = is_public;
        store_id = it->second.store_id;
        append_album_locked(store_id, it->second);
    }
    return true;
}

bool BinDb::is_album_public(const std::string& album_id) const {
    std::shared_lock lock(mutex_);
    auto it = albums_.find(album_id);
    if (it == albums_.end()) return false;
    return it->second.is_public;
}

bool BinDb::is_media_public(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return false;
    auto ait = albums_.find(it->second.album_id);
    if (ait == albums_.end()) return false;
    return ait->second.is_public;
}

std::vector<AlbumRecord> BinDb::list_public_albums() const {
    std::shared_lock lock(mutex_);
    std::vector<AlbumRecord> out;
    for (const auto& [id, album] : albums_) {
        if (album.is_public) out.push_back(album);
    }
    return out;
}

// ---- media ----

std::optional<MediaRecord> BinDb::add_media(const std::string& album_id,
                                             const std::string& filename,
                                             const std::string& content_type,
                                             std::vector<std::uint8_t> data) {
    std::optional<MediaRecord> result;
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto ait = albums_.find(album_id);
        if (ait == albums_.end()) return std::nullopt;

        store_id = ait->second.store_id;
        const fs::path p(filename);
        MediaRecord m;
        m.id = make_uuid();
        m.original_filename = filename;
        m.media_kind = detect_kind(p.extension().string());
        m.content_type = content_type;
        m.album_id = album_id;
        m.store_id = store_id;
        m.created_at = now_iso8601();
        m.size_bytes = data.size();
        m.pending_data = std::move(data);
        m.data_offset = 0;
        const std::string mid = m.id;
        media_[mid] = std::move(m);
        ait->second.media_ids.push_back(mid);

        append_media_locked(store_id, media_[mid]);
        result = media_[mid];
    }
    return result;
}

std::optional<MediaRecord> BinDb::get_media(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(id);
    if (it == media_.end()) return std::nullopt;
    return it->second;
}

std::vector<MediaRecord> BinDb::list_media(const std::string& album_id) const {
    std::shared_lock lock(mutex_);
    std::vector<MediaRecord> out;
    auto ait = albums_.find(album_id);
    if (ait == albums_.end()) return out;
    out.reserve(ait->second.media_ids.size());
    for (const auto& mid : ait->second.media_ids) {
        auto mit = media_.find(mid);
        if (mit != media_.end()) out.push_back(mit->second);
    }
    return out;
}

bool BinDb::delete_media(const std::string& id) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto mit = media_.find(id);
        if (mit == media_.end()) return false;
        store_id = mit->second.store_id;
        auto rec_off = mit->second.record_offset;
        auto ait = albums_.find(mit->second.album_id);
        if (ait != albums_.end()) {
            auto& mids = ait->second.media_ids;
            mids.erase(std::remove(mids.begin(), mids.end(), id), mids.end());
        }
        media_.erase(mit);
        zero_record_locked(store_id, rec_off);
    }
    return true;
}

// ---- read raw media data ----

std::vector<std::uint8_t> BinDb::read_media_data(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return {};
    const auto& m = it->second;
    if (m.size_bytes == 0) return {};
    if (!m.pending_data.empty()) return m.pending_data;

    auto mit = mmaps_.find(m.store_id);
    if (mit != mmaps_.end() && mit->second.valid()) {
        const auto& mm = mit->second;
        if (m.data_offset + m.size_bytes <= mm.size) {
            const auto* ptr = mm.data() + m.data_offset;
            return std::vector<std::uint8_t>(ptr, ptr + m.size_bytes);
        }
    }
    std::ifstream in(store_file(m.store_id), std::ios::binary);
    if (!in.is_open()) return {};
    in.seekg(static_cast<std::streamoff>(m.data_offset));
    std::vector<std::uint8_t> buf(m.size_bytes);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(m.size_bytes));
    if (static_cast<std::uint64_t>(in.gcount()) != m.size_bytes) return {};
    return buf;
}

std::vector<std::uint8_t> BinDb::read_media_data_range(const std::string& media_id,
                                                         std::uint64_t offset,
                                                         std::uint64_t length) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return {};
    const auto& m = it->second;
    if (m.size_bytes == 0 || offset >= m.size_bytes) return {};
    const std::uint64_t avail = m.size_bytes - offset;
    const std::uint64_t to_read = std::min(length, avail);

    if (!m.pending_data.empty()) {
        return std::vector<std::uint8_t>(
            m.pending_data.begin() + static_cast<std::ptrdiff_t>(offset),
            m.pending_data.begin() + static_cast<std::ptrdiff_t>(offset + to_read));
    }

    auto mit = mmaps_.find(m.store_id);
    if (mit != mmaps_.end() && mit->second.valid()) {
        const auto& mm = mit->second;
        const auto abs_off = m.data_offset + offset;
        if (abs_off + to_read <= mm.size) {
            const auto* ptr = mm.data() + abs_off;
            return std::vector<std::uint8_t>(ptr, ptr + to_read);
        }
    }
    std::ifstream in(store_file(m.store_id), std::ios::binary);
    if (!in.is_open()) return {};
    in.seekg(static_cast<std::streamoff>(m.data_offset + offset));
    std::vector<std::uint8_t> buf(to_read);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(to_read));
    if (static_cast<std::uint64_t>(in.gcount()) != to_read) return {};
    return buf;
}

std::uint64_t BinDb::get_media_size(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return 0;
    return it->second.size_bytes;
}

// ---- export ----

bool BinDb::export_db(const std::string& path) const {
    std::shared_lock lock(mutex_);
    const std::string tmp = path + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return false;
        out.write(EXPORT_MAGIC_V2, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(stores_.size()));

        for (const auto& [sid, store] : stores_) {
            bin_write_str(out, store.id);
            bin_write_str(out, store.name);
            bin_write_str(out, store.created_at);
            bin_write_u32(out, static_cast<std::uint32_t>(store.album_ids.size()));

            std::ifstream sin;
            constexpr std::size_t BUF = 1024 * 1024;
            std::vector<char> buf(BUF);

            for (const auto& aid : store.album_ids) {
                auto ait = albums_.find(aid);
                if (ait == albums_.end()) {
                    bin_write_str(out, aid);
                    bin_write_str(out, "");
                    bin_write_str(out, "");
                    { std::uint8_t pub = 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
                    bin_write_u32(out, 0);
                    continue;
                }
                const auto& album = ait->second;
                bin_write_str(out, album.id);
                bin_write_str(out, album.name);
                bin_write_str(out, album.created_at);
                { std::uint8_t pub = album.is_public ? 1 : 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
                bin_write_u32(out, static_cast<std::uint32_t>(album.media_ids.size()));

                for (const auto& mid : album.media_ids) {
                    auto mit = media_.find(mid);
                    if (mit == media_.end()) continue;
                    const auto& m = mit->second;

                    bin_write_str(out, m.id);
                    bin_write_str(out, m.original_filename);
                    bin_write_str(out, m.media_kind);
                    bin_write_str(out, m.content_type);
                    bin_write_str(out, m.created_at);
                    bin_write_u64(out, m.size_bytes);

                    if (!m.pending_data.empty()) {
                        out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                                  static_cast<std::streamsize>(m.pending_data.size()));
                    } else if (m.data_offset > 0 && m.size_bytes > 0) {
                        if (!sin.is_open())
                            sin.open(store_file(sid), std::ios::binary);
                        if (sin.is_open()) {
                            sin.seekg(static_cast<std::streamoff>(m.data_offset));
                            std::uint64_t remaining = m.size_bytes;
                            while (remaining > 0 && sin.good()) {
                                auto chunk = static_cast<std::streamsize>(
                                    std::min<std::uint64_t>(remaining, BUF));
                                sin.read(buf.data(), chunk);
                                auto got = sin.gcount();
                                if (got > 0) out.write(buf.data(), got);
                                remaining -= static_cast<std::uint64_t>(got);
                                if (got < chunk) break;
                            }
                        }
                    }
                }
            }
        }
    }
    fs::rename(tmp, path);
    return true;
}

std::vector<std::uint8_t> BinDb::export_db_to_buffer() const {
    std::shared_lock lock(mutex_);
    std::ostringstream oss(std::ios::binary);
    oss.write(EXPORT_MAGIC_V2, 4);

    auto wr_u32 = [&](std::uint32_t v) { oss.write(reinterpret_cast<const char*>(&v), 4); };
    auto wr_u64 = [&](std::uint64_t v) { oss.write(reinterpret_cast<const char*>(&v), 8); };
    auto wr_str = [&](const std::string& s) {
        auto len = static_cast<std::uint32_t>(s.size());
        wr_u32(len);
        if (len > 0) oss.write(s.data(), len);
    };

    wr_u32(static_cast<std::uint32_t>(stores_.size()));

    for (const auto& [sid, store] : stores_) {
        wr_str(store.id);
        wr_str(store.name);
        wr_str(store.created_at);
        wr_u32(static_cast<std::uint32_t>(store.album_ids.size()));

        std::ifstream sin;
        constexpr std::size_t BUF = 1024 * 1024;
        std::vector<char> buf(BUF);

        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) {
                wr_str(aid);
                wr_str("");
                wr_str("");
                { std::uint8_t pub = 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
                wr_u32(0);
                continue;
            }
            const auto& album = ait->second;
            wr_str(album.id);
            wr_str(album.name);
            wr_str(album.created_at);
            { std::uint8_t pub = album.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
            wr_u32(static_cast<std::uint32_t>(album.media_ids.size()));

            for (const auto& mid : album.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                const auto& m = mit->second;

                wr_str(m.id);
                wr_str(m.original_filename);
                wr_str(m.media_kind);
                wr_str(m.content_type);
                wr_str(m.created_at);
                wr_u64(m.size_bytes);

                if (!m.pending_data.empty()) {
                    oss.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                } else if (m.data_offset > 0 && m.size_bytes > 0) {
                    if (!sin.is_open())
                        sin.open(store_file(sid), std::ios::binary);
                    if (sin.is_open()) {
                        sin.seekg(static_cast<std::streamoff>(m.data_offset));
                        std::uint64_t remaining = m.size_bytes;
                        while (remaining > 0 && sin.good()) {
                            auto chunk = static_cast<std::streamsize>(
                                std::min<std::uint64_t>(remaining, BUF));
                            sin.read(buf.data(), chunk);
                            auto got = sin.gcount();
                            if (got > 0) oss.write(buf.data(), got);
                            remaining -= static_cast<std::uint64_t>(got);
                            if (got < chunk) break;
                        }
                    }
                }
            }
        }
    }

    std::string s = std::move(oss).str();
    return std::vector<std::uint8_t>(std::make_move_iterator(s.begin()), std::make_move_iterator(s.end()));
}

// ---- import ----

bool BinDb::import_from_stream(std::istream& in) {
    return import_from_stream(in, nullptr);
}

bool BinDb::import_from_stream(std::istream& in,
                                std::function<void(const std::string&, const uint8_t*, size_t)> on_replicate) {
    std::unique_lock lock(mutex_);

    char magic[4]{};
    in.read(magic, 4);
    const bool import_v2 = (std::memcmp(magic, EXPORT_MAGIC_V2, 4) == 0);
    if (!import_v2 && std::memcmp(magic, EXPORT_MAGIC, 4) != 0) {
        std::cerr << "[IMPORT] bad magic\n";
        return false;
    }

    // When streaming to cluster (on_replicate set), build the store buffer
    // directly from the input stream without loading media into pending_data.
    // This avoids keeping all media bytes in memory at once.
    const bool stream_to_cluster = (on_replicate != nullptr);

    auto num_stores = bin_read_u32(in);
    DBG_LOG("[IMPORT] start: " << num_stores << " stores, v2=" << import_v2
              << ", cluster=" << stream_to_cluster << "\n");
    std::size_t total_media_imported = 0;

    // Reusable I/O buffer — allocated once, grown as needed
    std::vector<char> import_buf;
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        std::string store_id = bin_read_str(in);
        std::string store_name = bin_read_str(in);
        std::string store_created = bin_read_str(in);
        DBG_LOG("[IMPORT] store " << (si+1) << "/" << num_stores
                  << " id=" << store_id << " name=" << store_name << "\n");

        bool new_store = (stores_.find(store_id) == stores_.end());
        if (new_store) {
            StoreRecord s;
            s.id = store_id;
            s.name = store_name;
            s.created_at = store_created;
            stores_[store_id] = s;
        }

        // Ensure MDS3 store file exists and open it once for all appends.
        // Open in update mode (in|out) and seek to end instead of using
        // ios::app: with ios::app the stream position indicator starts at 0
        // (only ios::ate seeks to end on open), so tellp() returns a
        // session-relative offset rather than the true file offset, which
        // corrupts the computed record/data offsets.
        std::ofstream store_out;
        if (!stream_to_cluster && !memory_only_) {
            ensure_store_file_locked(store_id);
            store_out.open(store_file(store_id), std::ios::binary | std::ios::in | std::ios::out);
            if (store_out.is_open()) store_out.seekp(0, std::ios::end);
        }

        // When streaming, build the MDS2 store buffer on the fly
        std::ostringstream store_buf;
        if (stream_to_cluster) {
            store_buf.write(STORE_MAGIC_V2, 4);
            // Placeholder for album count — will be patched below
            bin_write_u32(store_buf, 0);
        }
        std::uint32_t album_count = 0;

        auto num_albums = bin_read_u32(in);
        DBG_LOG("[IMPORT]   " << num_albums << " albums in store\n");
        for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
            std::string album_id = bin_read_str(in);
            std::string album_name = bin_read_str(in);
            std::string album_created = bin_read_str(in);

            bool album_is_public = false;
            if (import_v2) {
                std::uint8_t pub = 0;
                in.read(reinterpret_cast<char*>(&pub), 1);
                album_is_public = (pub != 0);
            }

            bool new_album = (albums_.find(album_id) == albums_.end());
            if (new_album) {
                AlbumRecord a;
                a.id = album_id;
                a.name = album_name;
                a.store_id = store_id;
                a.created_at = album_created;
                a.is_public = album_is_public;
                albums_[album_id] = a;
                stores_[store_id].album_ids.push_back(album_id);
                if (store_out.is_open()) {
                    // Write album record directly to open store file
                    std::ostringstream abody(std::ios::binary);
                    bin_write_str(abody, a.id);
                    bin_write_str(abody, a.name);
                    bin_write_str(abody, a.created_at);
                    { std::uint8_t pub = a.is_public ? 1 : 0;
                      abody.write(reinterpret_cast<const char*>(&pub), 1); }
                    std::string abody_str = abody.str();
                    auto aoff = static_cast<std::uint64_t>(store_out.tellp());
                    store_out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
                    auto absz = static_cast<std::uint64_t>(abody_str.size());
                    store_out.write(reinterpret_cast<const char*>(&absz), 8);
                    store_out.write(abody_str.data(), static_cast<std::streamsize>(abody_str.size()));
                    albums_[album_id].record_offset = aoff;
                }
            }

            // Start writing album to store buffer
            if (stream_to_cluster) {
                bin_write_str(store_buf, album_id);
                bin_write_str(store_buf, album_name);
                bin_write_str(store_buf, album_created);
                { std::uint8_t pub = album_is_public ? 1 : 0;
                  store_buf.write(reinterpret_cast<const char*>(&pub), 1); }
            }

            auto num_media = bin_read_u32(in);
            DBG_LOG("[IMPORT]   album " << (ai+1) << "/" << num_albums
                      << " \"" << album_name << "\" media=" << num_media << "\n");

            if (stream_to_cluster) {
                bin_write_u32(store_buf, num_media);
            }

            for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
                std::string media_id = bin_read_str(in);
                std::string orig_filename = bin_read_str(in);
                std::string media_kind = bin_read_str(in);
                std::string content_type = bin_read_str(in);
                std::string created_at = bin_read_str(in);
                std::uint64_t data_size = bin_read_u64(in);

                bool new_media = (media_.find(media_id) == media_.end());
                if ((mi + 1) % 50 == 0 || mi == 0 || mi + 1 == num_media) {
                    DBG_LOG("[IMPORT]     media " << (mi+1) << "/" << num_media
                              << " size=" << data_size << " new=" << new_media
                              << " stream_pos=" << in.tellg() << "\n");
                }

                if (stream_to_cluster) {
                    // Write media metadata to store buffer (NO raw data — stored separately)
                    bin_write_str(store_buf, media_id);
                    bin_write_str(store_buf, orig_filename);
                    bin_write_str(store_buf, media_kind);
                    bin_write_str(store_buf, content_type);
                    bin_write_str(store_buf, created_at);
                    bin_write_u64(store_buf, data_size);

                    if (data_size > 0) {
                        // In cluster mode, ALWAYS read and replicate media data:
                        // This allows re-importing an .mdb file to self-heal missing
                        // stripes and orphan blobs on nodes (e.g. from previous failures)
                        // even if the metadata itself is already present.
                        std::vector<std::uint8_t> media_data(data_size);
                        in.read(reinterpret_cast<char*>(media_data.data()),
                                static_cast<std::streamsize>(data_size));
                        auto got = static_cast<std::size_t>(in.gcount());
                        if (got < data_size) media_data.resize(got);
                        on_replicate("media:" + media_id,
                                     media_data.data(), media_data.size());
                    }

                    // Register metadata only (no raw data in RAM)
                    if (new_media) {
                        MediaRecord m;
                        m.id = media_id;
                        m.original_filename = orig_filename;
                        m.media_kind = media_kind;
                        m.content_type = content_type;
                        m.created_at = created_at;
                        m.size_bytes = data_size;
                        m.album_id = album_id;
                        m.store_id = store_id;
                        media_[media_id] = std::move(m);
                        albums_[album_id].media_ids.push_back(media_id);
                        ++total_media_imported;
                    }
                } else {
                    // Non-cluster import: stream directly to MDS3 store file
                    if (new_media) {
                        MediaRecord m;
                        m.id = media_id;
                        m.original_filename = orig_filename;
                        m.media_kind = media_kind;
                        m.content_type = content_type;
                        m.created_at = created_at;
                        m.size_bytes = data_size;
                        m.album_id = album_id;
                        m.store_id = store_id;

                        if (store_out.is_open()) {
                            // Compute metadata size without ostringstream
                            auto meta_size = static_cast<std::uint64_t>(
                                4 + m.id.size() + 4 + m.album_id.size() +
                                4 + m.original_filename.size() + 4 + m.media_kind.size() +
                                4 + m.content_type.size() + 4 + m.created_at.size() + 8);
                            auto total_body = meta_size + data_size;

                            // Write record header
                            m.record_offset = static_cast<std::uint64_t>(store_out.tellp());
                            store_out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
                            store_out.write(reinterpret_cast<const char*>(&total_body), 8);

                            // Write metadata fields directly
                            bin_write_str(store_out, m.id);
                            bin_write_str(store_out, m.album_id);
                            bin_write_str(store_out, m.original_filename);
                            bin_write_str(store_out, m.media_kind);
                            bin_write_str(store_out, m.content_type);
                            bin_write_str(store_out, m.created_at);
                            bin_write_u64(store_out, m.size_bytes);
                            m.data_offset = static_cast<std::uint64_t>(store_out.tellp());

                            // Stream raw data directly: input → store file (no RAM buffer)
                            if (data_size > 0) {
                                constexpr std::size_t BUF = 1024 * 1024;
                                if (import_buf.size() < std::min<std::size_t>(BUF, data_size))
                                    import_buf.resize(std::min<std::size_t>(BUF, data_size));
                                std::uint64_t remaining = data_size;
                                while (remaining > 0 && in.good()) {
                                    auto chunk = static_cast<std::streamsize>(
                                        std::min<std::uint64_t>(remaining, import_buf.size()));
                                    in.read(import_buf.data(), chunk);
                                    auto got = in.gcount();
                                    if (got > 0) store_out.write(import_buf.data(), got);
                                    remaining -= static_cast<std::uint64_t>(got);
                                    if (got < chunk) break;
                                }
                            }
                        } else {
                            // Memory-only mode: read media data into pending_data
                            if (data_size > 0) {
                                m.pending_data.resize(data_size);
                                in.read(reinterpret_cast<char*>(m.pending_data.data()),
                                        static_cast<std::streamsize>(data_size));
                                auto got = static_cast<std::uint64_t>(in.gcount());
                                if (got < data_size) m.pending_data.resize(got);
                            }
                        }

                        media_[media_id] = std::move(m);
                        albums_[album_id].media_ids.push_back(media_id);
                        ++total_media_imported;
                    } else {
                        if (data_size > 0) {
                            in.seekg(static_cast<std::streamoff>(data_size), std::ios::cur);
                        }
                    }
                }
            }

            album_count++;
        }

        // Non-cluster: close store file, then mmap
        if (store_out.is_open()) {
            store_out.close();
            mmap_store(store_id);
        }

        // Replicate metadata-only store buffer to cluster
        if (stream_to_cluster) {
            // Patch album count at offset 4 (after STORE_MAGIC_V2)
            std::string buf_str = store_buf.str();
            std::memcpy(buf_str.data() + 4, &album_count, 4);
            DBG_LOG("[IMPORT] replicate store:" << store_id
                      << " buf_size=" << buf_str.size() << "\n");
            on_replicate("store:" + store_id,
                         reinterpret_cast<const uint8_t*>(buf_str.data()),
                         buf_str.size());
            DBG_LOG("[IMPORT] replicate store:" << store_id << " done\n");

            // Save and replicate index after each store for crash consistency.
            // If import is interrupted, stores already replicated will have
            // a matching index referencing them.
            save_index_locked();
            auto idx_buf = save_index_to_buffer_locked();
            if (!idx_buf.empty()) {
                DBG_LOG("[IMPORT] replicate index size=" << idx_buf.size() << "\n");
                on_replicate("index", idx_buf.data(), idx_buf.size());
                DBG_LOG("[IMPORT] replicate index done\n");
            }
        }
    }

    // Vacuum touched stores to compact partial/duplicate records
    // from previous failed imports that used ios::app.
    if (!stream_to_cluster && !memory_only_) {
        for (const auto& [sid, _] : stores_)
            vacuum_store_locked(sid);
    }

    // Save index once after all stores are imported (not per-store)
    save_index_locked();

    // Replicate final index to cluster
    if (on_replicate) {
        auto buf = save_index_to_buffer_locked();
        if (!buf.empty()) {
            on_replicate("index", buf.data(), buf.size());
        }
    }

    return true;
}

bool BinDb::import_db(const std::string& path) {
    std::ifstream in(path, std::ios::binary);
    if (!in.is_open()) return false;
    return import_from_stream(in);
}

bool BinDb::import_db_from_buffer(const std::vector<std::uint8_t>& data) {
    return import_db_from_buffer(data.data(), data.size());
}

bool BinDb::import_db_from_buffer(const std::uint8_t* data, std::size_t len) {
    MemBuf mbuf(data, len);
    std::istream in(&mbuf);
    return import_from_stream(in);
}

bool BinDb::import_db_from_buffer(const std::uint8_t* data, std::size_t len,
                                   std::function<void(const std::string&, const uint8_t*, size_t)> on_replicate) {
    MemBuf mbuf(data, len);
    std::istream in(&mbuf);
    return import_from_stream(in, std::move(on_replicate));
}

// ---- Incremental import session (state machine fed with network chunks) ----

class BinDbImportSession : public ImportSession {
public:
    BinDbImportSession(BinDb& db,
                       ReplicateFn on_replicate,
                       BeginReplicateFn begin_replicate,
                       BinDb::WarmupFn warmup_fn = nullptr,
                       BinDb::StoreLifecycleFn on_store_begin = nullptr,
                       BinDb::StoreLifecycleFn on_store_end = nullptr)
        : db_(db), on_replicate_(std::move(on_replicate)),
          begin_replicate_(std::move(begin_replicate)),
          warmup_fn_(std::move(warmup_fn)),
          on_store_begin_(std::move(on_store_begin)),
          on_store_end_(std::move(on_store_end)),
          stream_to_cluster_(on_replicate_ != nullptr) {}

    ~BinDbImportSession() override {
        if (!done_.load(std::memory_order_acquire)) {
            // Aborted — close any open file
            if (store_out_.is_open()) store_out_.close();
        }
    }

    void feed(const char* data, size_t len) override {
        if (done_.load(std::memory_order_acquire)) return;
        std::cerr << "[IMPORT-SESSION] feed: len=" << len << " total_fed=" << (total_fed_ + len) << "\n";

        // Append incoming data under lock (once)
        {
            std::unique_lock lock(db_.mutex_);
            buf_.insert(buf_.end(), data, data + len);
            total_fed_ += len;
        }

        // Parse-drain loop: after each media entry or store finishes,
        // break out to drain deferred ops (finalize / replicate) so
        // each entry is fully replicated before the next one starts.
        for (;;) {
            {
                std::unique_lock lock(db_.mutex_);
                needs_drain_ = false;
                while (!done_.load(std::memory_order_relaxed) &&
                       !needs_drain_ && try_advance()) {}
                // Compact consumed bytes
                if (pos_ > 0) {
                    buf_.erase(buf_.begin(), buf_.begin() + static_cast<std::ptrdiff_t>(pos_));
                    pos_ = 0;
                }
            }
            // Execute deferred ops inline — waits for each replication
            drain_deferred();
            if (!needs_drain_ || done_.load(std::memory_order_acquire)) break;
        }

        std::cerr << "[IMPORT-SESSION] feed done: phase=" << static_cast<int>(phase_)
                  << " done=" << done_.load() << " ok=" << ok_.load() << "\n";
    }

    std::vector<std::function<void()>> take_deferred() override {
        std::lock_guard<std::mutex> lk(deferred_mutex_);
        std::vector<std::function<void()>> out;
        out.swap(pending_ops_);
        return out;
    }

    bool done() const override {
        return done_.load(std::memory_order_acquire);
    }
    bool ok() const override { return ok_.load(std::memory_order_acquire); }
    std::string error_message() const override {
        std::shared_lock lock(error_mutex_);
        return error_msg_;
    }

    Progress progress() const override {
        Progress p;
        p.bytes_fed = total_fed_;
        p.stores_total = num_stores_;
        p.stores_done = touched_stores_.size();
        p.media_total = num_media_;  // per-store count (resets per store)
        p.media_done = total_media_;

        if (done_.load(std::memory_order_acquire)) {
            p.phase = ok_.load(std::memory_order_acquire) ? "done" : "failed";
        } else if (parsing_done_.load(std::memory_order_acquire)) {
            p.phase = "replicating";
        } else {
            switch (phase_) {
            case Phase::MAGIC:
            case Phase::NUM_STORES:
                p.phase = "starting";
                break;
            case Phase::STORE_HEADER:
            case Phase::NUM_ALBUMS:
            case Phase::ALBUM_HEADER:
            case Phase::NUM_MEDIA:
                p.phase = "parsing";
                break;
            case Phase::MEDIA_HEADER:
            case Phase::MEDIA_DATA:
                p.phase = "importing";
                break;
            case Phase::DONE:
                p.phase = "finalizing";
                break;
            }
        }
        return p;
    }

private:
    // ---- read helpers (return false if not enough data) ----

    bool try_read_bytes(void* dest, size_t n) {
        if (buf_.size() - pos_ < n) return false;
        std::memcpy(dest, buf_.data() + pos_, n);
        pos_ += n;
        return true;
    }

    bool try_read_u32(std::uint32_t& out) { return try_read_bytes(&out, 4); }
    bool try_read_u64(std::uint64_t& out) { return try_read_bytes(&out, 8); }

    bool try_read_str(std::string& out) {
        if (buf_.size() - pos_ < 4) return false;
        std::uint32_t len;
        std::memcpy(&len, buf_.data() + pos_, 4);
        if (buf_.size() - pos_ < 4 + len) return false;
        out.assign(buf_.data() + pos_ + 4, len);
        pos_ += 4 + len;
        return true;
    }

    // ---- state machine ----

    bool try_advance() {
        switch (phase_) {

        case Phase::MAGIC: {
            char magic[4];
            if (!try_read_bytes(magic, 4)) return false;
            is_v2_ = (std::memcmp(magic, BinDb::EXPORT_MAGIC_V2, 4) == 0);
            if (!is_v2_ && std::memcmp(magic, BinDb::EXPORT_MAGIC, 4) != 0) {
                done_.store(true, std::memory_order_release);
                return false;
            }
            phase_ = Phase::NUM_STORES;
            return true;
        }

        case Phase::NUM_STORES: {
            if (!try_read_u32(num_stores_)) return false;
            std::cerr << "[IMPORT-SESSION] parsed num_stores=" << num_stores_ << "\n";
            si_ = 0;
            if (num_stores_ == 0) { finish_import(); return false; }
            phase_ = Phase::STORE_HEADER;
            return true;
        }

        case Phase::STORE_HEADER: {
            size_t saved = pos_;
            if (!try_read_str(cur_store_id_) ||
                !try_read_str(cur_store_name_) ||
                !try_read_str(cur_store_created_)) {
                pos_ = saved;
                return false;
            }
            // Create store if new
            if (db_.stores_.find(cur_store_id_) == db_.stores_.end()) {
                StoreRecord s;
                s.id = cur_store_id_;
                s.name = cur_store_name_;
                s.created_at = cur_store_created_;
                db_.stores_[cur_store_id_] = s;
            }
            // Open store file (non-cluster)
            if (!stream_to_cluster_ && !db_.memory_only_) {
                db_.ensure_store_file_locked(cur_store_id_);
                // Update mode + seek-to-end (not ios::app): ios::app leaves the
                // position indicator at 0 so tellp() would report a
                // session-relative offset, corrupting record/data offsets.
                store_out_.open(db_.store_file(cur_store_id_),
                                std::ios::binary | std::ios::in | std::ios::out);
                if (!store_out_.is_open()) {
                    fail("cannot open store file: " + db_.store_file(cur_store_id_).string());
                    return false;
                }
                store_out_.seekp(0, std::ios::end);
            }
            touched_stores_.insert(cur_store_id_);
            // Cluster: start store metadata buffer
            if (stream_to_cluster_) {
                if (on_store_begin_) {
                    std::lock_guard<std::mutex> dlk(deferred_mutex_);
                    pending_ops_.push_back(on_store_begin_);
                }
                store_meta_buf_.str("");
                store_meta_buf_.clear();
                store_meta_buf_.write(BinDb::STORE_MAGIC_V2, 4);
                bin_write_u32(store_meta_buf_, 0); // placeholder for album count
                store_album_count_ = 0;
            }
            phase_ = Phase::NUM_ALBUMS;
            return true;
        }

        case Phase::NUM_ALBUMS: {
            if (!try_read_u32(num_albums_)) return false;
            ai_ = 0;
            if (num_albums_ == 0) { finish_store(); return true; }
            phase_ = Phase::ALBUM_HEADER;
            return true;
        }

        case Phase::ALBUM_HEADER: {
            size_t saved = pos_;
            std::string album_id, album_name, album_created;
            if (!try_read_str(album_id) ||
                !try_read_str(album_name) ||
                !try_read_str(album_created)) {
                pos_ = saved;
                return false;
            }
            bool album_is_public = false;
            if (is_v2_) {
                std::uint8_t pub = 0;
                if (!try_read_bytes(&pub, 1)) { pos_ = saved; return false; }
                album_is_public = (pub != 0);
            }
            cur_album_id_ = album_id;
            bool new_album = (db_.albums_.find(album_id) == db_.albums_.end());
            if (new_album) {
                AlbumRecord a;
                a.id = album_id;
                a.name = album_name;
                a.store_id = cur_store_id_;
                a.created_at = album_created;
                a.is_public = album_is_public;
                db_.albums_[album_id] = a;
                db_.stores_[cur_store_id_].album_ids.push_back(album_id);
                // Write album record to store file
                if (store_out_.is_open()) {
                    std::ostringstream abody(std::ios::binary);
                    bin_write_str(abody, a.id);
                    bin_write_str(abody, a.name);
                    bin_write_str(abody, a.created_at);
                    { std::uint8_t pub = a.is_public ? 1 : 0;
                      abody.write(reinterpret_cast<const char*>(&pub), 1); }
                    std::string abody_str = abody.str();
                    auto aoff = static_cast<std::uint64_t>(store_out_.tellp());
                    store_out_.write(reinterpret_cast<const char*>(&BinDb::REC_ALBUM), 1);
                    auto absz = static_cast<std::uint64_t>(abody_str.size());
                    store_out_.write(reinterpret_cast<const char*>(&absz), 8);
                    store_out_.write(abody_str.data(), static_cast<std::streamsize>(abody_str.size()));
                    if (!store_out_.good()) {
                        fail("write error on store file for album " + album_id);
                        return false;
                    }
                    db_.albums_[album_id].record_offset = aoff;
                }
            }
            // Cluster: write album metadata
            if (stream_to_cluster_) {
                bin_write_str(store_meta_buf_, album_id);
                bin_write_str(store_meta_buf_, album_name);
                bin_write_str(store_meta_buf_, album_created);
                { std::uint8_t pub = album_is_public ? 1 : 0;
                  store_meta_buf_.write(reinterpret_cast<const char*>(&pub), 1); }
                ++store_album_count_;
            }
            phase_ = Phase::NUM_MEDIA;
            return true;
        }

        case Phase::NUM_MEDIA: {
            if (!try_read_u32(num_media_)) return false;
            if (stream_to_cluster_) bin_write_u32(store_meta_buf_, num_media_);
            mi_ = 0;
            if (num_media_ == 0) { advance_after_album(); return true; }
            phase_ = Phase::MEDIA_HEADER;
            return true;
        }

        case Phase::MEDIA_HEADER: {
            size_t saved = pos_;
            std::string media_id, orig_filename, media_kind, content_type, created_at;
            std::uint64_t data_size;
            if (!try_read_str(media_id) ||
                !try_read_str(orig_filename) ||
                !try_read_str(media_kind) ||
                !try_read_str(content_type) ||
                !try_read_str(created_at) ||
                !try_read_u64(data_size)) {
                pos_ = saved;
                return false;
            }
            cur_media_id_ = media_id;
            cur_media_new_ = (db_.media_.find(media_id) == db_.media_.end());
            data_size_ = data_size;
            data_written_ = 0;

            // Cluster: write media metadata to store buffer
            if (stream_to_cluster_) {
                bin_write_str(store_meta_buf_, media_id);
                bin_write_str(store_meta_buf_, orig_filename);
                bin_write_str(store_meta_buf_, media_kind);
                bin_write_str(store_meta_buf_, content_type);
                bin_write_str(store_meta_buf_, created_at);
                bin_write_u64(store_meta_buf_, data_size);
                // Begin a streaming replicate session for this media entry
                if (begin_replicate_ && data_size > 0) {
                    media_stream_ = begin_replicate_("media:" + media_id,
                                                     static_cast<size_t>(data_size));
                }
            }

            // Non-cluster: write media header to store file
            if (!stream_to_cluster_ && cur_media_new_ && store_out_.is_open()) {
                auto meta_size = static_cast<std::uint64_t>(
                    4 + media_id.size() + 4 + cur_album_id_.size() +
                    4 + orig_filename.size() + 4 + media_kind.size() +
                    4 + content_type.size() + 4 + created_at.size() + 8);
                auto total_body = meta_size + data_size;
                media_record_offset_ = static_cast<std::uint64_t>(store_out_.tellp());
                store_out_.write(reinterpret_cast<const char*>(&BinDb::REC_MEDIA), 1);
                store_out_.write(reinterpret_cast<const char*>(&total_body), 8);
                bin_write_str(store_out_, media_id);
                bin_write_str(store_out_, cur_album_id_);
                bin_write_str(store_out_, orig_filename);
                bin_write_str(store_out_, media_kind);
                bin_write_str(store_out_, content_type);
                bin_write_str(store_out_, created_at);
                bin_write_u64(store_out_, data_size);
                if (!store_out_.good()) {
                    fail("write error on store file for media " + media_id);
                    return false;
                }
                media_data_offset_ = static_cast<std::uint64_t>(store_out_.tellp());
            }

            // Stage metadata — do NOT insert into db_.media_ yet.
            // The record becomes visible only after all data is written
            // (in advance_after_media) to prevent concurrent readers from
            // seeing a media entry whose data is still being streamed.
            if (cur_media_new_) {
                MediaRecord m;
                m.id = media_id;
                m.original_filename = orig_filename;
                m.media_kind = media_kind;
                m.content_type = content_type;
                m.created_at = created_at;
                m.size_bytes = data_size;
                m.album_id = cur_album_id_;
                m.store_id = cur_store_id_;
                if (!stream_to_cluster_) {
                    m.record_offset = media_record_offset_;
                    m.data_offset = media_data_offset_;
                }
                pending_media_ = std::move(m);
                ++total_media_;
            }

            if (data_size == 0) { advance_after_media(); return true; }
            phase_ = Phase::MEDIA_DATA;
            return true;
        }

        case Phase::MEDIA_DATA: {
            drain_media_data();
            if (data_written_ >= data_size_) {
                advance_after_media();
                return true;
            }
            return false; // need more data
        }

        default:
            return false;
        }
    }

    void drain_media_data() {
        size_t avail = buf_.size() - pos_;
        size_t remaining = static_cast<size_t>(data_size_ - data_written_);
        size_t n = std::min(avail, remaining);
        if (n == 0) return;

        if (stream_to_cluster_) {
            // Stream media data directly to the cluster entry-by-entry.
            // Also accumulate in media_data_buf_ so we have a fallback
            // for one-shot replicate if the streaming finalize fails.
            if (media_stream_) {
                media_stream_->feed(
                    reinterpret_cast<const uint8_t*>(buf_.data() + pos_), n);
            }
            // Always accumulate — used as fallback on finalize failure,
            // or as primary path if no streaming session was created.
            media_data_buf_.insert(media_data_buf_.end(),
                                   buf_.data() + pos_,
                                   buf_.data() + pos_ + n);
        } else if (cur_media_new_) {
            if (store_out_.is_open()) {
                // Write directly to store file
                store_out_.write(buf_.data() + pos_, static_cast<std::streamsize>(n));
                if (!store_out_.good()) {
                    fail("write error on store file for store " + cur_store_id_);
                    return;
                }
            } else if (pending_media_) {
                // Memory-only mode: accumulate into pending_data
                pending_media_->pending_data.insert(
                    pending_media_->pending_data.end(),
                    buf_.data() + pos_,
                    buf_.data() + pos_ + n);
            }
        }
        // Advance pos_
        pos_ += n;
        data_written_ += n;
    }

    void fail(const std::string& msg) {
        {
            std::unique_lock lock(error_mutex_);
            error_msg_ = msg;
        }
        if (store_out_.is_open()) store_out_.close();
        phase_ = Phase::DONE;
        done_.store(true, std::memory_order_release);
    }

    void mark_media_cluster_unavailable(const std::string& mid, const std::string& reason) {
        auto it = db_.media_.find(mid);
        if (it != db_.media_.end()) {
            it->second.cluster_available = false;
        }
        std::cerr << "[IMPORT-SESSION] media degraded (cluster unavailable) id=" << mid
                  << " reason=" << reason << "\n";
    }

    void advance_after_media() {
        // Commit the media record now that all data is on disk.
        if (cur_media_new_ && pending_media_) {
            const std::string mid = pending_media_->id;
            const std::string aid = pending_media_->album_id;
            db_.media_[mid] = std::move(*pending_media_);
            db_.albums_[aid].media_ids.push_back(mid);
            pending_media_.reset();
        }

        // Cluster: finalize streaming session or fall back to one-shot replicate
        if (stream_to_cluster_) {
            if (media_stream_) {
                // Finalize the streaming session — computes parity + distributes.
                // On failure, warmup connections and retry via one-shot replicate
                // from the accumulated media_data_buf_.
                auto stream = std::move(media_stream_);
                std::string mid = cur_media_id_;
                auto fallback_data = std::make_shared<std::vector<uint8_t>>(
                    std::move(media_data_buf_));
                std::string fallback_key = "media:" + mid;
                {
                    std::lock_guard<std::mutex> dlk(deferred_mutex_);
                    pending_ops_.push_back([this, stream = std::shared_ptr<ReplicateSession>(std::move(stream)),
                                           mid, fallback_data, fallback_key]() {
                        stream->finalize();
                        if (stream->ok()) return;
                        std::cerr << "[IMPORT-SESSION] finalize failed for " << mid
                                  << ": " << stream->error_message()
                                  << " — warmup + one-shot fallback\n";
                        if (warmup_fn_) warmup_fn_();
                        if (!on_replicate_(fallback_key, fallback_data->data(), fallback_data->size())) {
                            // Keep import successful: data is stored locally, but mark
                            // this media unavailable in cluster until next repair/import.
                            mark_media_cluster_unavailable(
                                mid, "replicate failed for " + fallback_key +
                                     " (after finalize failure)");
                        }
                    });
                }
                needs_drain_ = true;
            } else if (!media_data_buf_.empty()) {
                // No streaming session — one-shot replicate
                auto data = std::make_shared<std::vector<uint8_t>>(
                    std::move(media_data_buf_));
                std::string key = "media:" + cur_media_id_;
                std::string mid = cur_media_id_;
                {
                    std::lock_guard<std::mutex> dlk(deferred_mutex_);
                    pending_ops_.push_back([this, key, mid, data]() {
                        if (!on_replicate_(key, data->data(), data->size())) {
                            mark_media_cluster_unavailable(mid,
                                "replicate failed for " + key);
                        }
                    });
                }
                needs_drain_ = true;
            }
        }
        media_data_buf_.clear();
        
        ++mi_;
        if (mi_ < num_media_) {
            phase_ = Phase::MEDIA_HEADER;
        } else {
            advance_after_album();
        }
    }

    void advance_after_album() {
        ++ai_;
        if (ai_ < num_albums_) {
            phase_ = Phase::ALBUM_HEADER;
        } else {
            finish_store();
        }
    }

    void finish_store() {
        if (store_out_.is_open()) {
            store_out_.close();
            db_.mmap_store(cur_store_id_);
        }
        if (stream_to_cluster_) {
            // Patch album count and defer replication (outside db_.mutex_)
            std::string buf_str = store_meta_buf_.str();
            std::memcpy(buf_str.data() + 4, &store_album_count_, 4);
            auto data = std::make_shared<std::vector<uint8_t>>(buf_str.begin(), buf_str.end());
            std::string key = "store:" + cur_store_id_;
            {
                std::lock_guard<std::mutex> dlk(deferred_mutex_);
                pending_ops_.push_back([this, key, data]() {
                    if (!on_replicate_(key, data->data(), data->size()))
                        fail("replicate failed for " + key);
                });
                if (on_store_end_)
                    pending_ops_.push_back(on_store_end_);
            }
            needs_drain_ = true; // wait for store metadata replication
        }
        ++si_;
        if (si_ < num_stores_) {
            phase_ = Phase::STORE_HEADER;
        } else {
            finish_import();
        }
    }

    void finish_import() {
        std::cerr << "[IMPORT-SESSION] finish_import: touched_stores=" << touched_stores_.size()
                  << " total_media=" << total_media_ << " total_fed=" << total_fed_
                  << " stream_to_cluster=" << stream_to_cluster_ << "\n";
        // Vacuum touched stores to compact partial/duplicate records
        // from previous failed imports that used ios::app.
        if (!stream_to_cluster_) {
            for (const auto& sid : touched_stores_)
                db_.vacuum_store_locked(sid);
        }
        db_.save_index_locked();
        if (on_replicate_) {
            auto buf = std::make_shared<std::vector<uint8_t>>(
                db_.save_index_to_buffer_locked());
            std::cerr << "[IMPORT-SESSION] finish_import: index_buf_size=" << buf->size() << "\n";
            if (!buf->empty()) {
                std::lock_guard<std::mutex> dlk(deferred_mutex_);
                pending_ops_.push_back([this, buf]() {
                    std::cerr << "[IMPORT-SESSION] replicating index (" << buf->size() << " bytes)\n";
                    if (!on_replicate_("index", buf->data(), buf->size()))
                        fail("replicate failed for index");
                });
            }
        }
        // Mark parsing complete — done_/ok_ are set AFTER deferred
        // cluster replications finish (in drain_deferred_final).
        phase_ = Phase::DONE;
        parsing_done_.store(true, std::memory_order_release);
    }

    // ---- members ----

    BinDb& db_;
    ReplicateFn on_replicate_;
    BeginReplicateFn begin_replicate_;
    BinDb::WarmupFn warmup_fn_;
    BinDb::StoreLifecycleFn on_store_begin_;
    BinDb::StoreLifecycleFn on_store_end_;
    bool stream_to_cluster_;

    std::vector<char> buf_;
    size_t pos_ = 0;
    bool needs_drain_ = false; // break parsing loop to drain deferred ops per entry

    enum class Phase {
        MAGIC, NUM_STORES,
        STORE_HEADER, NUM_ALBUMS,
        ALBUM_HEADER, NUM_MEDIA,
        MEDIA_HEADER, MEDIA_DATA,
        DONE
    } phase_ = Phase::MAGIC;

    bool is_v2_ = false;
    std::uint32_t num_stores_ = 0, si_ = 0;
    std::uint32_t num_albums_ = 0, ai_ = 0;
    std::uint32_t num_media_ = 0, mi_ = 0;

    std::string cur_store_id_, cur_store_name_, cur_store_created_;
    std::string cur_album_id_;
    std::string cur_media_id_;
    bool cur_media_new_ = false;
    std::uint64_t data_size_ = 0, data_written_ = 0;
    std::uint64_t media_record_offset_ = 0, media_data_offset_ = 0;
    std::optional<MediaRecord> pending_media_;

    // Non-cluster: store file handle
    std::ofstream store_out_;

    // Cluster: per-store metadata buffer (no raw data)
    std::ostringstream store_meta_buf_{std::ios::binary};
    std::uint32_t store_album_count_ = 0;

    // Cluster: per-media streaming session (streams directly to cluster)
    std::unique_ptr<ReplicateSession> media_stream_;

    // Cluster: per-media binary data accumulator (fallback if no streaming session)
    std::vector<uint8_t> media_data_buf_;

    // Deferred cluster operations dispatched outside db_.mutex_
    std::mutex deferred_mutex_;
    std::vector<std::function<void()>> pending_ops_;

    void drain_deferred() {
        auto ops = take_deferred();
        if (!ops.empty())
            std::cerr << "[IMPORT-SESSION] drain_deferred: executing " << ops.size() << " ops\n";
        for (auto& op : ops) op();
        // All deferred cluster replications finished — now mark done
        if (parsing_done_.load(std::memory_order_acquire) &&
            !done_.load(std::memory_order_relaxed)) {
            // Check no new deferred ops were queued during execution
            std::lock_guard<std::mutex> dlk(deferred_mutex_);
            if (pending_ops_.empty()) {
                std::cerr << "[IMPORT-SESSION] drain_deferred: marking done+ok\n";
                ok_.store(true, std::memory_order_release);
                done_.store(true, std::memory_order_release);
            }
        }
    }

    std::atomic<bool> done_{false}, ok_{false};
    std::atomic<bool> parsing_done_{false};
    mutable std::shared_mutex error_mutex_;
    std::string error_msg_;
    size_t total_media_ = 0;
    size_t total_fed_ = 0;
    std::set<std::string> touched_stores_;
};

std::unique_ptr<ImportSession> BinDb::begin_import() {
    return std::make_unique<BinDbImportSession>(*this, nullptr, nullptr);
}

std::unique_ptr<ImportSession> BinDb::begin_import(
    ReplicateFn on_replicate,
    BeginReplicateFn begin_replicate,
    WarmupFn warmup_fn,
    StoreLifecycleFn on_store_begin,
    StoreLifecycleFn on_store_end) {
    return std::make_unique<BinDbImportSession>(*this, std::move(on_replicate),
                                                 std::move(begin_replicate),
                                                 std::move(warmup_fn),
                                                 std::move(on_store_begin),
                                                 std::move(on_store_end));
}

// ---- reload / store_ids ----

void BinDb::reload() {
    if (memory_only_) return;  // memory-only mode has no files to reload from
    std::unique_lock lock(mutex_);
    stores_.clear();
    albums_.clear();
    media_.clear();
    mmaps_.clear();
    domain_label_index_.clear();
    load_index();
    for (auto& [sid, store] : stores_) {
        load_store(sid, store);
    }
    rebuild_domain_label_index_locked();
    mmap_all_stores();
}

std::vector<std::string> BinDb::store_ids() const {
    std::shared_lock lock(mutex_);
    std::vector<std::string> ids;
    ids.reserve(stores_.size());
    for (const auto& [k, _] : stores_) ids.push_back(k);
    return ids;
}

std::vector<std::string> BinDb::media_ids() const {
    std::shared_lock lock(mutex_);
    std::vector<std::string> ids;
    ids.reserve(media_.size());
    for (const auto& [k, _] : media_) ids.push_back(k);
    return ids;
}

// ---- load ----

void BinDb::load() {
    if (memory_only_) return;
    load_index();
    for (auto& [sid, store] : stores_) {
        load_store(sid, store);
    }
    rebuild_domain_label_index_locked();
    mmap_all_stores();
}

void BinDb::rebuild_domain_label_index_locked() {
    domain_label_index_.clear();
    for (const auto& [sid, store] : stores_) {
        for (const auto& d : store.auth_domains) {
            domain_label_index_[d.label] = {sid, d.id};
        }
    }
}

void BinDb::load_index() {
    std::ifstream in(index_file(), std::ios::binary);
    if (!in.is_open()) return;

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v4 = (std::memcmp(magic, INDEX_MAGIC, 4) == 0);
    const bool is_v3 = is_v4 || (std::memcmp(magic, INDEX_MAGIC_V3, 4) == 0);
    if (!is_v3 && std::memcmp(magic, INDEX_MAGIC_V2, 4) != 0) return;

    auto num_stores = bin_read_u32(in);
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        StoreRecord s;
        s.id = bin_read_str(in);
        s.name = bin_read_str(in);
        s.created_at = bin_read_str(in);
        if (is_v3) {
            auto num_domains = bin_read_u32(in);
            for (std::uint32_t di = 0; di < num_domains && in.good(); ++di) {
                AuthDomain d;
                d.id = bin_read_str(in);
                d.label = bin_read_str(in);
                d.url = bin_read_str(in);
                d.client_name = bin_read_str(in);
                d.client_secret = bin_read_str(in);
                s.auth_domains.push_back(std::move(d));
            }
        }
        if (is_v4) {
            auto num_acls = bin_read_u32(in);
            for (std::uint32_t ai = 0; ai < num_acls && in.good(); ++ai) {
                StoreACL acl;
                acl.id = bin_read_str(in);
                acl.group_id = bin_read_str(in);
                acl.group_name = bin_read_str(in);
                std::uint8_t flags = 0;
                in.read(reinterpret_cast<char*>(&flags), 1);
                acl.can_read  = (flags & 0x01) != 0;
                acl.can_write = (flags & 0x02) != 0;
                s.acls.push_back(std::move(acl));
            }
        }
        stores_[s.id] = std::move(s);
    }
}

void BinDb::load_store(const std::string& store_id, StoreRecord& store) {
    std::ifstream in(store_file(store_id), std::ios::binary);
    if (!in.is_open()) return;

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v3 = (std::memcmp(magic, STORE_MAGIC_V3, 4) == 0);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v3 && !is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    if (is_v3) {
        // MDS3: flat record stream with 4 bytes reserved padding after magic
        in.seekg(8);
        while (in.good()) {
            auto rec_off = static_cast<std::uint64_t>(in.tellg());
            std::uint8_t rec_type = 0;
            in.read(reinterpret_cast<char*>(&rec_type), 1);
            if (!in.good()) break;
            std::uint64_t body_size = 0;
            in.read(reinterpret_cast<char*>(&body_size), 8);
            if (!in.good()) break;

            // Next record starts after header (9 bytes) + body
            auto next_rec = static_cast<std::streamoff>(rec_off + 9 + body_size);

            if (rec_type == REC_ALBUM) {
                AlbumRecord a;
                a.id = bin_read_str(in);
                a.name = bin_read_str(in);
                a.created_at = bin_read_str(in);
                { std::uint8_t pub = 0;
                  in.read(reinterpret_cast<char*>(&pub), 1);
                  a.is_public = (pub != 0); }
                a.store_id = store_id;
                a.record_offset = rec_off;

                // Later album records with same ID overwrite earlier ones (for edits)
                auto existing = albums_.find(a.id);
                if (existing != albums_.end()) {
                    a.media_ids = std::move(existing->second.media_ids);
                    albums_[a.id] = std::move(a);
                } else {
                    const std::string aid = a.id;
                    albums_[aid] = std::move(a);
                    store.album_ids.push_back(aid);
                }
            } else if (rec_type == REC_MEDIA) {
                MediaRecord m;
                m.id = bin_read_str(in);
                m.album_id = bin_read_str(in);
                m.original_filename = bin_read_str(in);
                m.media_kind = bin_read_str(in);
                m.content_type = bin_read_str(in);
                m.created_at = bin_read_str(in);
                m.size_bytes = bin_read_u64(in);
                m.store_id = store_id;
                m.record_offset = rec_off;
                m.data_offset = static_cast<std::uint64_t>(in.tellg());
                // Skip raw data — don't read it into memory

                auto ait = albums_.find(m.album_id);
                if (ait != albums_.end()) {
                    auto& mids = ait->second.media_ids;
                    if (std::find(mids.begin(), mids.end(), m.id) == mids.end()) {
                        mids.push_back(m.id);
                    }
                }
                media_[m.id] = std::move(m);
            }
            // Jump to next record (skips REC_EMPTY, unknown types,
            // and any remaining body bytes not consumed above)
            in.seekg(next_rec);
        }

        // Second pass: link orphaned media to their albums.
        // When set_album_public() zeros + re-appends an album record,
        // media records may appear before their album in the file.
        // Those media are stored in media_ but not yet in album.media_ids.
        for (const auto& [mid, m] : media_) {
            if (m.store_id != store_id) continue;
            auto ait = albums_.find(m.album_id);
            if (ait == albums_.end()) continue;
            auto& mids = ait->second.media_ids;
            if (std::find(mids.begin(), mids.end(), mid) == mids.end()) {
                mids.push_back(mid);
            }
        }

        return;
    }

    // MDS1/MDS2: hierarchical format (legacy)
    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;
            m.data_offset = static_cast<std::uint64_t>(in.tellg());
            in.seekg(static_cast<std::streamoff>(m.size_bytes), std::ios::cur);

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        const std::string aid = a.id;
        store.album_ids.push_back(aid);
        albums_[aid] = std::move(a);
    }
}

// ---- save ----

void BinDb::save_index_locked() const {
    if (memory_only_) return;
    const std::string tmp = index_file().string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;
        out.write(INDEX_MAGIC, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(stores_.size()));
        for (const auto& [sid, s] : stores_) {
            bin_write_str(out, s.id);
            bin_write_str(out, s.name);
            bin_write_str(out, s.created_at);
            bin_write_u32(out, static_cast<std::uint32_t>(s.auth_domains.size()));
            for (const auto& d : s.auth_domains) {
                bin_write_str(out, d.id);
                bin_write_str(out, d.label);
                bin_write_str(out, d.url);
                bin_write_str(out, d.client_name);
                bin_write_str(out, d.client_secret);
            }
            bin_write_u32(out, static_cast<std::uint32_t>(s.acls.size()));
            for (const auto& acl : s.acls) {
                bin_write_str(out, acl.id);
                bin_write_str(out, acl.group_id);
                bin_write_str(out, acl.group_name);
                std::uint8_t flags = 0;
                if (acl.can_read)  flags |= 0x01;
                if (acl.can_write) flags |= 0x02;
                out.write(reinterpret_cast<const char*>(&flags), 1);
            }
        }
    }
    fs::rename(tmp, index_file());
}

void BinDb::save_store_locked(const std::string& store_id) {
    if (memory_only_) return;  // in memory-only mode, data stays in pending_data
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    const auto& store = sit->second;

    const fs::path sf = store_file(store_id);
    std::ifstream old_in(sf, std::ios::binary);

    const std::string tmp = sf.string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;
        out.write(STORE_MAGIC_V2, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(store.album_ids.size()));

        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) continue;
            const auto& a = ait->second;

            bin_write_str(out, a.id);
            bin_write_str(out, a.name);
            bin_write_str(out, a.created_at);
            { std::uint8_t pub = a.is_public ? 1 : 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
            bin_write_u32(out, static_cast<std::uint32_t>(a.media_ids.size()));

            for (const auto& mid : a.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                auto& m = mit->second;

                bin_write_str(out, m.id);
                bin_write_str(out, m.original_filename);
                bin_write_str(out, m.media_kind);
                bin_write_str(out, m.content_type);
                bin_write_str(out, m.created_at);
                bin_write_u64(out, m.size_bytes);

                const auto new_offset = static_cast<std::uint64_t>(out.tellp());

                if (!m.pending_data.empty()) {
                    out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                    m.pending_data.clear();
                    m.pending_data.shrink_to_fit();
                } else if (old_in.is_open() && m.data_offset > 0 && m.size_bytes > 0) {
                    old_in.seekg(static_cast<std::streamoff>(m.data_offset));
                    constexpr std::size_t BUF = 1024 * 1024;
                    std::vector<char> buf(BUF);
                    std::uint64_t remaining = m.size_bytes;
                    while (remaining > 0 && old_in.good()) {
                        auto chunk = static_cast<std::streamsize>(std::min<std::uint64_t>(remaining, BUF));
                        old_in.read(buf.data(), chunk);
                        auto got = old_in.gcount();
                        if (got > 0) out.write(buf.data(), got);
                        remaining -= static_cast<std::uint64_t>(got);
                        if (got < chunk) break;
                    }
                }

                m.data_offset = new_offset;
            }
        }
    }
    old_in.close();
    fs::rename(tmp, sf);
    mmap_store(store_id);
}

// ---- MDS3: append-only store operations ----

void BinDb::ensure_store_file_locked(const std::string& store_id) {
    if (memory_only_) return;
    if (store_id.empty()) return;
    const fs::path sf = store_file(store_id);
    if (fs::exists(sf)) return;
    std::ofstream out(sf, std::ios::binary | std::ios::trunc);
    if (!out.is_open()) return;
    out.write(STORE_MAGIC_V3, 4);
    std::uint32_t reserved = 0;
    out.write(reinterpret_cast<const char*>(&reserved), 4);
}

void BinDb::append_album_locked(const std::string& store_id, const AlbumRecord& album) {
    if (memory_only_) return;
    ensure_store_file_locked(store_id);
    const fs::path sf = store_file(store_id);
    // Update mode + seek-to-end (not ios::app) so tellp() yields the true
    // file offset for record_offset (ios::app starts the position at 0).
    std::ofstream out(sf, std::ios::binary | std::ios::in | std::ios::out);
    if (!out.is_open()) return;
    out.seekp(0, std::ios::end);

    // Build album body
    std::ostringstream body(std::ios::binary);
    bin_write_str(body, album.id);
    bin_write_str(body, album.name);
    bin_write_str(body, album.created_at);
    { std::uint8_t pub = album.is_public ? 1 : 0;
      body.write(reinterpret_cast<const char*>(&pub), 1); }
    std::string body_str = body.str();

    // Record offset is the current end of file
    auto rec_off = static_cast<std::uint64_t>(out.tellp());

    // Write record header: type(u8) + body_size(u64)
    out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
    auto bsz = static_cast<std::uint64_t>(body_str.size());
    out.write(reinterpret_cast<const char*>(&bsz), 8);
    out.write(body_str.data(), static_cast<std::streamsize>(body_str.size()));

    // Update in-memory record offset
    auto ait = albums_.find(album.id);
    if (ait != albums_.end()) ait->second.record_offset = rec_off;

    out.close();
    mmap_store(store_id);
}

void BinDb::append_media_locked(const std::string& store_id, MediaRecord& media) {
    if (memory_only_) return;
    ensure_store_file_locked(store_id);
    const fs::path sf = store_file(store_id);
    // Update mode + seek-to-end (not ios::app) so tellp() yields the true
    // file offset for record_offset/data_offset (ios::app starts position at 0).
    std::ofstream out(sf, std::ios::binary | std::ios::in | std::ios::out);
    if (!out.is_open()) return;
    out.seekp(0, std::ios::end);

    // Build media body (metadata portion)
    std::ostringstream meta(std::ios::binary);
    bin_write_str(meta, media.id);
    bin_write_str(meta, media.album_id);
    bin_write_str(meta, media.original_filename);
    bin_write_str(meta, media.media_kind);
    bin_write_str(meta, media.content_type);
    bin_write_str(meta, media.created_at);
    bin_write_u64(meta, media.size_bytes);
    std::string meta_str = meta.str();

    auto body_size = static_cast<std::uint64_t>(meta_str.size()) + media.size_bytes;

    // Record offset
    auto rec_off = static_cast<std::uint64_t>(out.tellp());

    // Write record header: type(u8) + body_size(u64)
    out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
    out.write(reinterpret_cast<const char*>(&body_size), 8);

    // Write metadata
    out.write(meta_str.data(), static_cast<std::streamsize>(meta_str.size()));

    // data_offset = current file position (start of raw data)
    auto data_off = static_cast<std::uint64_t>(out.tellp());

    // Write raw data
    if (!media.pending_data.empty()) {
        out.write(reinterpret_cast<const char*>(media.pending_data.data()),
                  static_cast<std::streamsize>(media.pending_data.size()));
        media.pending_data.clear();
        media.pending_data.shrink_to_fit();
    }

    media.record_offset = rec_off;
    media.data_offset = data_off;

    out.close();
    mmap_store(store_id);
}

void BinDb::zero_record_locked(const std::string& store_id, std::uint64_t record_offset) {
    if (memory_only_) return;
    if (record_offset == 0) return;
    const fs::path sf = store_file(store_id);
    int fd = ::open(sf.c_str(), O_RDWR);
    if (fd < 0) return;

    // Mark as deleted: single byte write (type = REC_EMPTY)
    // body_size stays intact so load/vacuum can skip over the record
    std::uint8_t empty_type = REC_EMPTY;
    if (::pwrite(fd, &empty_type, 1, static_cast<off_t>(record_offset)) < 0) {
        ::close(fd);
        return;
    }

    ::close(fd);
    mmap_store(store_id);
}

void BinDb::vacuum_store_locked(const std::string& store_id) {
    if (memory_only_) return;
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    const auto& store = sit->second;
    const fs::path sf = store_file(store_id);
    if (!fs::exists(sf)) return;

    const std::string tmp = sf.string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;

        // Write MDS3 header
        out.write(STORE_MAGIC_V3, 4);
        std::uint32_t reserved = 0;
        out.write(reinterpret_cast<const char*>(&reserved), 4);

        std::ifstream old_in(sf, std::ios::binary);

        // Write all valid albums
        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) continue;
            auto& a = ait->second;

            std::ostringstream body(std::ios::binary);
            bin_write_str(body, a.id);
            bin_write_str(body, a.name);
            bin_write_str(body, a.created_at);
            { std::uint8_t pub = a.is_public ? 1 : 0;
              body.write(reinterpret_cast<const char*>(&pub), 1); }
            std::string body_str = body.str();

            a.record_offset = static_cast<std::uint64_t>(out.tellp());
            out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
            auto bsz = static_cast<std::uint64_t>(body_str.size());
            out.write(reinterpret_cast<const char*>(&bsz), 8);
            out.write(body_str.data(), static_cast<std::streamsize>(body_str.size()));

            // Write all valid media in this album
            for (const auto& mid : a.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                auto& m = mit->second;

                std::ostringstream mbody(std::ios::binary);
                bin_write_str(mbody, m.id);
                bin_write_str(mbody, m.album_id);
                bin_write_str(mbody, m.original_filename);
                bin_write_str(mbody, m.media_kind);
                bin_write_str(mbody, m.content_type);
                bin_write_str(mbody, m.created_at);
                bin_write_u64(mbody, m.size_bytes);
                std::string mbody_str = mbody.str();

                auto total_body = static_cast<std::uint64_t>(mbody_str.size()) + m.size_bytes;

                m.record_offset = static_cast<std::uint64_t>(out.tellp());
                out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
                out.write(reinterpret_cast<const char*>(&total_body), 8);
                out.write(mbody_str.data(), static_cast<std::streamsize>(mbody_str.size()));

                auto old_data_offset = m.data_offset; // save before vacuum overwrites

                m.data_offset = static_cast<std::uint64_t>(out.tellp());

                // Copy raw data from old file or pending_data
                if (!m.pending_data.empty()) {
                    out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                    m.pending_data.clear();
                    m.pending_data.shrink_to_fit();
                } else if (old_in.is_open() && m.size_bytes > 0 && old_data_offset > 0) {
                    old_in.seekg(static_cast<std::streamoff>(old_data_offset));
                    constexpr std::size_t BUF = 1024 * 1024;
                    std::vector<char> buf(BUF);
                    std::uint64_t rem = m.size_bytes;
                    while (rem > 0 && old_in.good()) {
                        auto chunk = static_cast<std::streamsize>(std::min<std::uint64_t>(rem, BUF));
                        old_in.read(buf.data(), chunk);
                        auto got = old_in.gcount();
                        if (got > 0) out.write(buf.data(), got);
                        rem -= static_cast<std::uint64_t>(got);
                        if (got < chunk) break;
                    }
                }
            }
        }
    }
    fs::rename(tmp, sf);
    mmap_store(store_id);
}

void BinDb::vacuum_store(const std::string& store_id) {
    std::unique_lock lock(mutex_);
    vacuum_store_locked(store_id);
}

void BinDb::vacuum_all_stores() {
    std::unique_lock lock(mutex_);
    for (const auto& [sid, _] : stores_) {
        vacuum_store_locked(sid);
    }
}

// ---- mmap management ----

void BinDb::mmap_store(const std::string& store_id) {
    auto path = store_file(store_id);
    if (fs::exists(path) && fs::file_size(path) > 0) {
        mmaps_[store_id] = MmapRegion::open(path);
    }
}

void BinDb::mmap_all_stores() {
    for (const auto& [sid, _] : stores_) {
        mmap_store(sid);
    }
}

// ---- buffer-based save/load (for cluster-only storage) ----

std::vector<std::uint8_t> BinDb::save_index_to_buffer() const {
    std::shared_lock lock(mutex_);
    return save_index_to_buffer_locked();
}

std::vector<std::uint8_t> BinDb::save_index_to_buffer_locked() const {
    std::ostringstream oss(std::ios::binary);
    oss.write(INDEX_MAGIC, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(stores_.size()));
    for (const auto& [sid, s] : stores_) {
        bin_write_str(oss, s.id);
        bin_write_str(oss, s.name);
        bin_write_str(oss, s.created_at);
        bin_write_u32(oss, static_cast<std::uint32_t>(s.auth_domains.size()));
        for (const auto& d : s.auth_domains) {
            bin_write_str(oss, d.id);
            bin_write_str(oss, d.label);
            bin_write_str(oss, d.url);
            bin_write_str(oss, d.client_name);
            bin_write_str(oss, d.client_secret);
        }
        bin_write_u32(oss, static_cast<std::uint32_t>(s.acls.size()));
        for (const auto& acl : s.acls) {
            bin_write_str(oss, acl.id);
            bin_write_str(oss, acl.group_id);
            bin_write_str(oss, acl.group_name);
            std::uint8_t flags = 0;
            if (acl.can_read)  flags |= 0x01;
            if (acl.can_write) flags |= 0x02;
            oss.write(reinterpret_cast<const char*>(&flags), 1);
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_index_from_buffer(const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v4 = (std::memcmp(magic, INDEX_MAGIC, 4) == 0);
    const bool is_v3 = is_v4 || (std::memcmp(magic, INDEX_MAGIC_V3, 4) == 0);
    if (!is_v3 && std::memcmp(magic, INDEX_MAGIC_V2, 4) != 0) return;

    // Preserve existing store IDs to detect removed stores
    std::set<std::string> old_ids;
    for (const auto& [k, _] : stores_) old_ids.insert(k);

    auto num_stores = bin_read_u32(in);
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        StoreRecord s;
        s.id = bin_read_str(in);
        s.name = bin_read_str(in);
        s.created_at = bin_read_str(in);
        if (is_v3) {
            auto num_domains = bin_read_u32(in);
            for (std::uint32_t di = 0; di < num_domains && in.good(); ++di) {
                AuthDomain d;
                d.id = bin_read_str(in);
                d.label = bin_read_str(in);
                d.url = bin_read_str(in);
                d.client_name = bin_read_str(in);
                d.client_secret = bin_read_str(in);
                s.auth_domains.push_back(std::move(d));
            }
        }
        if (is_v4) {
            auto num_acls = bin_read_u32(in);
            for (std::uint32_t ai = 0; ai < num_acls && in.good(); ++ai) {
                StoreACL acl;
                acl.id = bin_read_str(in);
                acl.group_id = bin_read_str(in);
                acl.group_name = bin_read_str(in);
                std::uint8_t flags = 0;
                in.read(reinterpret_cast<char*>(&flags), 1);
                acl.can_read  = (flags & 0x01) != 0;
                acl.can_write = (flags & 0x02) != 0;
                s.acls.push_back(std::move(acl));
            }
        }
        // Keep existing album_ids if store already present
        auto existing = stores_.find(s.id);
        if (existing != stores_.end()) {
            s.album_ids = existing->second.album_ids;
        }
        stores_[s.id] = std::move(s);
        old_ids.erase(s.id);
    }

    // Do not remove locally-present stores that are absent from the buffer.
    // Deletes propagate explicitly via delete_store() + cluster_.remove();
    // removing here would wipe out recently-created local stores when the
    // synced index is stale (same merge-only approach as authdb).
    rebuild_domain_label_index_locked();
}

std::vector<std::uint8_t> BinDb::save_store_to_buffer(const std::string& store_id) {
    std::shared_lock lock(mutex_);
    return save_store_to_buffer_locked(store_id);
}

std::vector<std::uint8_t> BinDb::save_store_to_buffer_locked(const std::string& store_id) const {
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return {};
    const auto& store = sit->second;

    std::ostringstream oss(std::ios::binary);
    oss.write(STORE_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(store.album_ids.size()));

    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait == albums_.end()) continue;
        const auto& a = ait->second;

        bin_write_str(oss, a.id);
        bin_write_str(oss, a.name);
        bin_write_str(oss, a.created_at);
        { std::uint8_t pub = a.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
        bin_write_u32(oss, static_cast<std::uint32_t>(a.media_ids.size()));

        for (const auto& mid : a.media_ids) {
            auto mit = media_.find(mid);
            if (mit == media_.end()) continue;
            const auto& m = mit->second;

            bin_write_str(oss, m.id);
            bin_write_str(oss, m.original_filename);
            bin_write_str(oss, m.media_kind);
            bin_write_str(oss, m.content_type);
            bin_write_str(oss, m.created_at);
            bin_write_u64(oss, m.size_bytes);

            // In memory-only mode, data is always in pending_data
            if (!m.pending_data.empty()) {
                oss.write(reinterpret_cast<const char*>(m.pending_data.data()),
                          static_cast<std::streamsize>(m.pending_data.size()));
            }
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_store_from_buffer(const std::string& store_id, const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    auto& store = sit->second;

    // Clear old albums/media for this store
    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) {
            for (const auto& mid : ait->second.media_ids) media_.erase(mid);
            albums_.erase(ait);
        }
    }
    store.album_ids.clear();

    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;

            // Read media binary data into pending_data (memory-only)
            if (m.size_bytes > 0) {
                m.pending_data.resize(m.size_bytes);
                in.read(reinterpret_cast<char*>(m.pending_data.data()),
                        static_cast<std::streamsize>(m.size_bytes));
            }
            m.data_offset = 0;

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        const std::string aid = a.id;
        store.album_ids.push_back(aid);
        albums_[aid] = std::move(a);
    }
}

// ==================== ClusterMediaBackend ====================

// ---- metadata-only store buffer (no raw media data) ----

std::vector<std::uint8_t> BinDb::save_store_metadata_to_buffer(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    return save_store_metadata_to_buffer_locked(store_id);
}

std::vector<std::uint8_t> BinDb::save_store_metadata_to_buffer_locked(const std::string& store_id) const {
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return {};
    const auto& store = sit->second;

    std::ostringstream oss(std::ios::binary);
    oss.write(STORE_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(store.album_ids.size()));

    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait == albums_.end()) continue;
        const auto& a = ait->second;

        bin_write_str(oss, a.id);
        bin_write_str(oss, a.name);
        bin_write_str(oss, a.created_at);
        { std::uint8_t pub = a.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
        bin_write_u32(oss, static_cast<std::uint32_t>(a.media_ids.size()));

        for (const auto& mid : a.media_ids) {
            auto mit = media_.find(mid);
            if (mit == media_.end()) continue;
            const auto& m = mit->second;

            bin_write_str(oss, m.id);
            bin_write_str(oss, m.original_filename);
            bin_write_str(oss, m.media_kind);
            bin_write_str(oss, m.content_type);
            bin_write_str(oss, m.created_at);
            bin_write_u64(oss, m.size_bytes);
            // NO raw data written — metadata only
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_store_metadata_from_buffer(const std::string& store_id, const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    auto& store = sit->second;

    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    // Merge-only: never remove existing albums/media that aren't in the
    // incoming buffer.  This prevents stale cluster data from overwriting
    // freshly imported or uploaded content.
    std::set<std::string> incoming_album_ids;
    std::set<std::string> incoming_media_ids;

    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;
            m.data_offset = 0;

            incoming_media_ids.insert(m.id);

            // Only add media that doesn't already exist locally
            if (media_.find(m.id) == media_.end()) {
                a.media_ids.push_back(m.id);
                media_[m.id] = std::move(m);
            }
        }

        incoming_album_ids.insert(a.id);

        auto existing = albums_.find(a.id);
        if (existing != albums_.end()) {
            // Album exists — merge: keep existing media_ids, add new ones
            // from the incoming data that we just inserted into media_.
            std::set<std::string> existing_set(existing->second.media_ids.begin(),
                                                existing->second.media_ids.end());
            for (const auto& mid : a.media_ids) {
                if (existing_set.find(mid) == existing_set.end()) {
                    existing->second.media_ids.push_back(mid);
                }
            }
            // Update metadata fields
            existing->second.name = a.name;
            existing->second.is_public = a.is_public;
        } else {
            // New album — add and register in store
            const std::string aid = a.id;
            albums_[aid] = std::move(a);
            if (std::find(store.album_ids.begin(), store.album_ids.end(), aid) == store.album_ids.end()) {
                store.album_ids.push_back(aid);
            }
        }
    }
}

std::optional<MediaRecord> BinDb::add_media_metadata(const std::string& album_id,
                                                      const std::string& filename,
                                                      const std::string& content_type,
                                                      std::uint64_t size_bytes) {
    std::unique_lock lock(mutex_);
    auto ait = albums_.find(album_id);
    if (ait == albums_.end()) return std::nullopt;

    const fs::path p(filename);
    MediaRecord m;
    m.id = make_uuid();
    m.original_filename = filename;
    m.media_kind = detect_kind(p.extension().string());
    m.content_type = content_type;
    m.album_id = album_id;
    m.store_id = ait->second.store_id;
    m.created_at = now_iso8601();
    m.size_bytes = size_bytes;
    m.data_offset = 0;
    // no pending_data — raw data goes directly to cluster

    const std::string mid = m.id;
    media_[mid] = std::move(m);
    ait->second.media_ids.push_back(mid);

    auto mit = media_.find(mid);
    if (mit == media_.end()) return std::nullopt;
    return mit->second;
}

ClusterMediaBackend::ClusterMediaBackend(Cluster& cluster, BlobCache& cache)
    : local_(), cluster_(cluster), cache_(cache) {}

void ClusterMediaBackend::start_sync() {
    // Retry initial sync with a total time budget of 30 seconds.
    // If the cluster layer is slow (connection timeouts, stripe retries),
    // don't block server startup indefinitely — the background sync_loop
    // will keep retrying.
    auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
    for (int attempt = 1; attempt <= 5; ++attempt) {
        sync_from_cluster();
        // Require store metadata to have loaded too, not just the index —
        // otherwise a run where every store's metadata fetch failed would
        // still report "OK" here with 0 albums loaded (initial_sync_ok_
        // only reflects the index itself, set well before the per-store
        // loop runs).
        if (initial_sync_ok_.load() && last_sync_stores_complete_.load()) {
            std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
            auto sids = local_.store_ids();
            std::size_t total_albums = 0;
            for (const auto& sid : sids)
                total_albums += local_.list_albums(sid).size();
            std::cerr << "[CLUSTER-SYNC] initial sync OK on attempt " << attempt
                      << ": " << sids.size() << " stores, " << total_albums << " albums\n";
            break;
        }
        if (std::chrono::steady_clock::now() >= deadline) {
            std::cerr << "[CLUSTER-SYNC] initial sync time budget exhausted after "
                      << attempt << " attempts, continuing in background\n";
            break;
        }
        std::cerr << "[CLUSTER-SYNC] initial sync attempt " << attempt
                  << "/5 incomplete, retrying in 2s\n";
        std::this_thread::sleep_for(std::chrono::seconds(2));
    }
    {
        std::lock_guard<std::mutex> lock(sync_mutex_);
        sync_requested_ = false;
        repair_requested_ = false;
        sync_ticket_ = 0;
        sync_done_ticket_ = 0;
    }
    sync_running_ = true;
    sync_thread_ = std::thread(&ClusterMediaBackend::sync_loop, this);
}

ClusterMediaBackend::~ClusterMediaBackend() {
    sync_running_ = false;
    sync_cv_.notify_all();
    if (sync_thread_.joinable()) sync_thread_.join();
}

void ClusterMediaBackend::request_sync(bool wait_for_completion) {
    std::unique_lock<std::mutex> lock(sync_mutex_);
    sync_requested_ = true;
    const std::uint64_t ticket = ++sync_ticket_;
    sync_cv_.notify_all();
    if (wait_for_completion) {
        sync_cv_.wait(lock, [this, ticket] {
            return !sync_running_.load() || sync_done_ticket_ >= ticket;
        });
    }
}

bool ClusterMediaBackend::import_active() const {
    return importing_.load() || cluster_.isImportRunning();
}

// ---- stores ----
StoreRecord ClusterMediaBackend::create_store(const std::string& name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto s = local_.create_store(name);
    replicate_store(s.id);
    replicate_index();
    return s;
}

std::optional<StoreRecord> ClusterMediaBackend::get_store(const std::string& id) const {
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto s = local_.get_store(id);
        if (s) return s;
    }
    const_cast<ClusterMediaBackend*>(this)->sync_from_cluster();
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_store(id);
}

std::vector<StoreRecord> ClusterMediaBackend::list_stores() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_stores();
}

bool ClusterMediaBackend::delete_store(const std::string& id) {
    if (!initial_sync_ok_.load()) {
        throw std::runtime_error("Cluster node isolated or syncing. Try again later.");
    }
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    
    // Collect all media and albums to remove their cluster objects
    std::vector<std::string> media_to_remove;
    auto albums = local_.list_albums(id);
    for (const auto& album : albums) {
        auto media_list = local_.list_media(album.id);
        for (const auto& m : media_list) {
            media_to_remove.push_back(m.id);
        }
    }

    if (!local_.delete_store(id)) return false;

    // Remove store metadata blob
    cluster_.remove("store:" + id);
    
    // Remove all associated media data blobs
    for (const auto& mid : media_to_remove) {
        cluster_.remove("media:" + mid);
    }
    
    // Record tombstone so other nodes also delete this store on sync
    tombstones_.insert(id);
    replicate_tombstones();
    replicate_index(true);  // force: store count decreased intentionally
    return true;
}

bool ClusterMediaBackend::rename_store(const std::string& id, const std::string& new_name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.rename_store(id, new_name)) return false;
    replicate_index();
    return true;
}

// ---- store auth domains ----
std::string ClusterMediaBackend::add_store_domain(const std::string& store_id, AuthDomain domain) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto did = local_.add_store_domain(store_id, std::move(domain));
    if (!did.empty()) replicate_index();
    return did;
}

bool ClusterMediaBackend::remove_store_domain(const std::string& store_id, const std::string& domain_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.remove_store_domain(store_id, domain_id)) return false;
    replicate_index();
    return true;
}

std::vector<AuthDomain> ClusterMediaBackend::list_store_domains(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_store_domains(store_id);
}

std::optional<AuthDomain> ClusterMediaBackend::find_domain_by_label(const std::string& label) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.find_domain_by_label(label);
}

// ---- store ACLs ----
std::string ClusterMediaBackend::add_store_acl(const std::string& store_id, StoreACL acl) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto aid = local_.add_store_acl(store_id, std::move(acl));
    if (!aid.empty()) replicate_index();
    return aid;
}

bool ClusterMediaBackend::remove_store_acl(const std::string& store_id, const std::string& acl_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.remove_store_acl(store_id, acl_id)) return false;
    replicate_index();
    return true;
}

std::vector<StoreACL> ClusterMediaBackend::list_store_acls(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_store_acls(store_id);
}

std::vector<std::string> ClusterMediaBackend::all_acl_group_ids() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.all_acl_group_ids();
}

// ---- albums ----
std::optional<AlbumRecord> ClusterMediaBackend::create_album(const std::string& store_id, const std::string& name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto a = local_.create_album(store_id, name);
    if (a) replicate_store(store_id);
    return a;
}

std::optional<AlbumRecord> ClusterMediaBackend::get_album(const std::string& id) const {
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto a = local_.get_album(id);
        if (a) return a;
    }
    const_cast<ClusterMediaBackend*>(this)->sync_from_cluster();
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_album(id);
}

std::vector<AlbumRecord> ClusterMediaBackend::list_albums(const std::string& store_id) const {
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto albums = local_.list_albums(store_id);
        if (!albums.empty()) return albums;
    }
    // Empty locally — try a fresh sync in case store metadata hasn't arrived yet
    const_cast<ClusterMediaBackend*>(this)->sync_from_cluster();
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_albums(store_id);
}

bool ClusterMediaBackend::delete_album(const std::string& id) {
    if (!initial_sync_ok_.load()) {
        throw std::runtime_error("Cluster node isolated or syncing. Try again later.");
    }
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto album = local_.get_album(id);
    if (!album) return false;
    std::string store_id = album->store_id;
    if (!local_.delete_album(id)) return false;
    // Re-replicate store (album and its media removed from inline blob)
    replicate_store(store_id);
    replicate_index();
    return true;
}

bool ClusterMediaBackend::set_album_public(const std::string& album_id, bool is_public) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto album = local_.get_album(album_id);
    if (!album) return false;
    if (!local_.set_album_public(album_id, is_public)) return false;
    replicate_store(album->store_id);
    return true;
}

bool ClusterMediaBackend::is_album_public(const std::string& album_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.is_album_public(album_id);
}

bool ClusterMediaBackend::is_media_public(const std::string& media_id) const {
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        if (local_.get_media(media_id)) return local_.is_media_public(media_id);
    }
    const_cast<ClusterMediaBackend*>(this)->sync_from_cluster();
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.is_media_public(media_id);
}

std::vector<AlbumRecord> ClusterMediaBackend::list_public_albums() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_public_albums();
}

// ---- media ----
std::optional<MediaRecord> ClusterMediaBackend::add_media(const std::string& album_id,
                                                            const std::string& filename,
                                                            const std::string& content_type,
                                                            std::vector<std::uint8_t> data) {
    if (!initial_sync_ok_.load()) {
        throw std::runtime_error("Cluster node isolated or syncing. Try again later.");
    }
    std::optional<MediaRecord> m;
    {
        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        // Store raw data separately, keep only metadata in local BinDb
        std::vector<std::uint8_t> raw_data = data;
        m = local_.add_media(album_id, filename, content_type, std::move(data));
        if (m) {
            // Replicate media data as separate key — retry on failure
            if (!raw_data.empty()) {
                bool ok = cluster_.replicate("media:" + m->id, raw_data.data(), raw_data.size());
                if (!ok) {
                    // Retry once after warmup
                    cluster_.warmup_read_clients();
                    ok = cluster_.replicate("media:" + m->id, raw_data.data(), raw_data.size());
                }
                if (!ok) {
                    std::cerr << "[CLUSTER] add_media: replicate FAILED for media:" << m->id
                              << " (" << raw_data.size() << " bytes) — rolling back\n";
                    local_.delete_media(m->id);
                    return std::nullopt;
                }
            }
            // Replicate store metadata (no inline media data)
            replicate_store(m->store_id);
            replicate_index();
        }
    }
    // Queue an immediate sync instead of relying on an operator to press
    // "Sync from Cluster" (or the periodic timer). Must be non-blocking:
    // sync_from_cluster() waits for the dedicated sync thread to finish a
    // full cycle, which can hang for a long time while the cluster is
    // degraded (index/store fetches retry with multi-second deadlines) —
    // that would tie up this HTTP worker thread instead of just the one
    // background sync thread, and with enough concurrent uploads it can
    // exhaust the whole worker pool. request_sync(false) only wakes the
    // background thread and returns immediately.
    if (m) request_sync(false);
    return m;
}

std::optional<MediaRecord> ClusterMediaBackend::get_media(const std::string& id) const {
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto m = local_.get_media(id);
        if (m) return m;
    }
    // Only attempt a re-sync if the last one wasn't too recent (avoids
    // blocking an HTTP request for 30+ s with a full cluster sync).
    auto now = std::chrono::steady_clock::now();
    if (now - last_on_demand_sync_.load() > std::chrono::seconds(10)) {
        last_on_demand_sync_.store(now);
        const_cast<ClusterMediaBackend*>(this)->sync_from_cluster();
    }
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_media(id);
}

std::vector<MediaRecord> ClusterMediaBackend::list_media(const std::string& album_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto items = local_.list_media(album_id);
    if (!cluster_.isRunning()) return items;
    // Tag each media record with cluster availability using cached peer group data.
    auto peer_groups = cluster_.list_peer_groups();
    std::unordered_set<uint64_t> known_gids;
    for (const auto& pg : peer_groups) {
        if (pg.online) {
            known_gids.insert(pg.groups.begin(), pg.groups.end());
        }
    }
    for (auto& m : items) {
        uint64_t gid = cluster_group_id("media:" + m.id);
        m.cluster_available = (known_gids.count(gid) > 0)
                              || !m.pending_data.empty();
    }
    return items;
}

bool ClusterMediaBackend::delete_media(const std::string& id) {
    if (!initial_sync_ok_.load()) {
        throw std::runtime_error("Cluster node isolated or syncing. Try again later.");
    }
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto media = local_.get_media(id);
    if (!media) return false;
    std::string store_id = media->store_id;
    if (!local_.delete_media(id)) return false;
    // Remove media data from cluster
    cluster_.remove("media:" + id);
    // Re-replicate store metadata
    replicate_store(store_id);
    replicate_index();
    return true;
}

// ---- raw data access (per-media cluster keys) ----

std::vector<std::uint8_t> ClusterMediaBackend::read_media_data(const std::string& media_id) const {
    // Try local pending_data first (from add_media before replicate)
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto result = local_.read_media_data(media_id);
        if (!result.empty()) return result;
    }
    // Fetch media data directly from cluster (per-media key)
    std::vector<uint8_t> out;
    const_cast<Cluster&>(cluster_).fetch("media:" + media_id, out);
    return out;
}

std::vector<std::uint8_t> ClusterMediaBackend::read_media_data_range(const std::string& media_id,
                                                                       std::uint64_t offset,
                                                                       std::uint64_t length) const {
    // Try local pending_data first
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto result = local_.read_media_data_range(media_id, offset, length);
        if (!result.empty()) return result;
    }
    // Fetch through LRU cache (avoids repeated cluster round-trips)
    return cache_fetch_range(media_id, offset, length);
}

std::vector<std::uint8_t> ClusterMediaBackend::cache_fetch_range(const std::string& media_id,
                                                                   std::uint64_t offset,
                                                                   std::uint64_t length) const {
    std::string key = "data:" + media_id + ":" + std::to_string(offset) + ":" + std::to_string(length);

    // Check cache
    auto cached = cache_.get(key);
    if (cached) return cached->data;

    // Cache miss — fetch from cluster
    std::vector<uint8_t> out;
    const_cast<Cluster&>(cluster_).fetch_range("media:" + media_id, offset, length, out);
    if (out.empty()) return out;

    // Take a copy for the caller, then move the original into the cache.
    std::vector<uint8_t> result(out);
    BlobValue blob;
    blob.data = std::move(out);
    blob.type = BlobType::data;
    cache_.put(key, std::move(blob));
    return result;
}

std::uint64_t ClusterMediaBackend::get_media_size(const std::string& media_id) const {
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto size = local_.get_media_size(media_id);
        if (size > 0) return size;
    }
    // Only attempt a re-sync if the last one wasn't too recent
    auto now = std::chrono::steady_clock::now();
    if (now - last_on_demand_sync_.load() > std::chrono::seconds(10)) {
        last_on_demand_sync_.store(now);
        const_cast<ClusterMediaBackend*>(this)->sync_from_cluster();
    }
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_media_size(media_id);
}

// ---- export / import ----
bool ClusterMediaBackend::export_db(const std::string& path) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    // Export requires raw data — fetch each media from cluster
    auto buf = export_db_to_buffer_impl();
    if (buf.empty()) return false;
    std::ofstream out(path, std::ios::binary | std::ios::trunc);
    if (!out.is_open()) return false;
    out.write(reinterpret_cast<const char*>(buf.data()),
              static_cast<std::streamsize>(buf.size()));
    return out.good();
}

std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return export_db_to_buffer_impl();
}

std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer_impl() const {
    // Build export buffer with raw data fetched from cluster
    static constexpr char EXPORT_MAGIC_V2[4] = {'M', 'D', 'E', '2'};

    // Pass 1: gather the store/album/media tree (cheap local metadata reads)
    // and every media id that needs a cluster fetch.
    struct AlbumEntry {
        AlbumRecord album;
        std::vector<MediaRecord> media;
    };
    struct StoreEntry {
        StoreRecord store;
        std::vector<AlbumEntry> albums;
    };
    std::vector<StoreEntry> tree;
    std::vector<std::string> media_ids;

    for (auto& store : local_.list_stores()) {
        StoreEntry se{store, {}};
        for (auto& album : local_.list_albums(store.id)) {
            AlbumEntry ae{album, local_.list_media(album.id)};
            for (auto& m : ae.media) media_ids.push_back(m.id);
            se.albums.push_back(std::move(ae));
        }
        tree.push_back(std::move(se));
    }

    // Pass 2: fetch every media's raw data concurrently. Previously this
    // fetched one media at a time inline while writing the export buffer,
    // so a store with many items — or any item hitting per-node retry
    // delays — could make the whole HTTP request run far longer than any
    // client-side timeout. Cluster::fetch() round-robins across a pool of
    // independent read connections (READ_POOL_SIZE), so a handful of
    // fetches genuinely run in parallel instead of queueing behind each
    // other.
    std::unordered_map<std::string, std::vector<std::uint8_t>> fetched;
    fetched.reserve(media_ids.size());
    {
        std::mutex fetched_mutex;
        static constexpr size_t MAX_CONCURRENT = 4;
        auto fetch_one = [&](const std::string& id) {
            std::vector<std::uint8_t> raw;
            const_cast<Cluster&>(cluster_).fetch("media:" + id, raw);
            std::lock_guard<std::mutex> lk(fetched_mutex);
            fetched.emplace(id, std::move(raw));
        };
        for (size_t next = 0; next < media_ids.size(); ) {
            size_t batch_end = std::min(next + MAX_CONCURRENT, media_ids.size());
            std::vector<std::future<void>> futures;
            for (size_t i = next; i < batch_end; ++i)
                futures.push_back(std::async(std::launch::async, fetch_one, media_ids[i]));
            for (auto& f : futures) f.get();
            next = batch_end;
        }
    }

    // Pass 3: write the export in the original binary layout, using the
    // pre-fetched data instead of fetching inline.
    static const std::vector<std::uint8_t> kEmpty;
    std::ostringstream oss(std::ios::binary);
    oss.write(EXPORT_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(tree.size()));

    for (const auto& se : tree) {
        bin_write_str(oss, se.store.id);
        bin_write_str(oss, se.store.name);
        bin_write_str(oss, se.store.created_at);
        bin_write_u32(oss, static_cast<std::uint32_t>(se.albums.size()));

        for (const auto& ae : se.albums) {
            bin_write_str(oss, ae.album.id);
            bin_write_str(oss, ae.album.name);
            bin_write_str(oss, ae.album.created_at);
            { std::uint8_t pub = ae.album.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
            bin_write_u32(oss, static_cast<std::uint32_t>(ae.media.size()));

            for (const auto& m : ae.media) {
                bin_write_str(oss, m.id);
                bin_write_str(oss, m.original_filename);
                bin_write_str(oss, m.media_kind);
                bin_write_str(oss, m.content_type);
                bin_write_str(oss, m.created_at);

                auto it = fetched.find(m.id);
                const std::vector<std::uint8_t>& raw = (it != fetched.end()) ? it->second : kEmpty;
                // Write actual fetched size — not m.size_bytes — so the
                // header always matches the data that follows. If the
                // fetch failed the entry gets size 0 (metadata preserved,
                // no corrupt zero-padding).
                bin_write_u64(oss, static_cast<std::uint64_t>(raw.size()));
                if (!raw.empty()) {
                    oss.write(reinterpret_cast<const char*>(raw.data()),
                              static_cast<std::streamsize>(raw.size()));
                }
            }
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

bool ClusterMediaBackend::import_db(const std::string& path) {
    // Read the file into a buffer and use import_db_from_buffer which
    // correctly replicates both media data ("media:<id>") and metadata
    // ("store:<id>", "index") to the cluster.  The old code only called
    // replicate_store (metadata only), leaving media data on the local
    // node only.
    std::ifstream in(path, std::ios::binary | std::ios::ate);
    if (!in.is_open()) return false;
    auto size = static_cast<std::size_t>(in.tellg());
    in.seekg(0, std::ios::beg);
    std::vector<std::uint8_t> buf(size);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(size));
    if (static_cast<std::size_t>(in.gcount()) != size) return false;
    return import_db_from_buffer(buf.data(), buf.size());
}

bool ClusterMediaBackend::import_db_from_buffer(const std::vector<std::uint8_t>& data) {
    return import_db_from_buffer(data.data(), data.size());
}

bool ClusterMediaBackend::import_db_from_buffer(const std::uint8_t* data, std::size_t len) {
    std::cerr << "[CLUSTER-IMPORT] start, input_size=" << len << "\n";
    if (!cluster_.isRunning()) {
        std::cerr << "[CLUSTER-IMPORT] cluster not running!\n";
        return false;
    }

    importing_.store(true);
    struct ImportGuard {
        std::atomic<bool>& flag;
        ~ImportGuard() { flag.store(false); }
    } import_guard{importing_};

    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);

    // Warmup cluster clients once before starting.
    // Use the same pclient path as normal uploads for stability.
    cluster_.warmup_read_clients();

    // Synchronous replicate callback: each entry is written to cluster
    // immediately as it is parsed, using the same pclient path as uploads.
    int repl_count = 0;
    int repl_fail = 0;
    auto replicate_fn = [&](const std::string& key, const uint8_t* d, size_t l) {
        ++repl_count;
        if (repl_count % 50 == 1)
            std::cerr << "[CLUSTER-IMPORT] replicate #" << repl_count
                      << " key=" << key << " size=" << l << "\n";

        constexpr int MAX_RETRIES = 3;
        for (int attempt = 0; attempt < MAX_RETRIES; ++attempt) {
            if (attempt > 0) {
                cluster_.warmup_read_clients();
                std::this_thread::sleep_for(
                    std::chrono::milliseconds(300 * (1 << attempt)));
            }
            bool ok = cluster_.replicate(key, d, l);
            if (ok) return;
        }
        std::cerr << "[CLUSTER-IMPORT] FAILED key=" << key << "\n";
        ++repl_fail;
    };

    bool parse_ok = local_.import_db_from_buffer(data, len, replicate_fn);
    std::cerr << "[CLUSTER-IMPORT] done, ok=" << parse_ok
              << " replicated=" << repl_count << " failed=" << repl_fail << "\n";
    if (!parse_ok || repl_fail > 0) return false;

    // Clear ALL tombstones after successful import. A full import replaces
    // the entire dataset — any previous delete tombstones are stale and must
    // not suppress the newly imported stores on other nodes.
    if (!tombstones_.empty()) {
        tombstones_.clear();
        replicate_tombstones();
        std::cerr << "[CLUSTER-IMPORT] cleared all tombstones\n";
    }

    return true;
}

std::unique_ptr<ImportSession> ClusterMediaBackend::begin_import() {
    std::cerr << "[CLUSTER-IMPORT] begin_import: cluster_running=" << cluster_.isRunning() << "\n";
    if (!cluster_.isRunning()) {
        std::cerr << "[CLUSTER-IMPORT] begin_import: cluster not running!\n";
        return nullptr;
    }

    importing_.store(true);

    // Synchronous replicate: use pclient_ for all keys, same path as uploads.
    auto replicate_fn = [this](const std::string& key, const uint8_t* d, size_t len) -> bool {
        const bool is_media = (key.compare(0, 6, "media:") == 0);

        // Fast path first: do not pre-clean every media key. That pre-clean
        // adds avoidable latency for healthy writes and slows large imports.
        if (cluster_.replicate(key, d, len)) return true;

        // Retry policy tuned for import throughput:
        // - media keys: one short recovery retry after cleanup/warmup
        // - non-media keys (store/index): a small number of retries
        const int retries = is_media ? 1 : 2;
        for (int attempt = 1; attempt <= retries; ++attempt) {
            std::cerr << "[CLUSTER-IMPORT-STREAM] replicate retry " << attempt
                      << " key=" << key << " size=" << len << "\n";

            if (is_media) {
                // Cleanup only on failure, not unconditionally per media.
                cluster_.remove(key);
            }
            cluster_.warmup_read_clients();

            // Keep waits short to avoid multi-minute stalls in import.
            std::this_thread::sleep_for(std::chrono::milliseconds(150 * attempt));
            if (cluster_.replicate(key, d, len)) return true;
        }

        std::cerr << "[CLUSTER-IMPORT-STREAM] replicate FAILED key=" << key << "\n";
        return false;
    };

    // Disable streaming begin_store/finalize for import media. One-shot
    // replicate() follows the stable upload path and avoids import-only
    // finalize timeouts.
    auto begin_replicate_fn = [this](const std::string& key, size_t total_size)
            -> std::unique_ptr<ReplicateSession> {
        (void)key;
        (void)total_size;
        return nullptr;
    };

    auto warmup_fn = [this]() {
        cluster_.warmup_read_clients();
    };

    auto session = local_.begin_import(std::move(replicate_fn), std::move(begin_replicate_fn),
                                       std::move(warmup_fn), nullptr, nullptr);
    if (!session) {
        importing_.store(false);
        return nullptr;
    }

    // Wrap to handle cleanup on completion
    class ImportGuardSession : public ImportSession {
    public:
        ImportGuardSession(std::unique_ptr<ImportSession> inner,
                           std::atomic<bool>& flag,
                           ClusterMediaBackend& backend)
            : inner_(std::move(inner)), flag_(flag), backend_(backend) {}
        ~ImportGuardSession() override {
            if (ok()) {
                // Clear ALL tombstones after successful import. Any previous
                // delete tombstones are stale and must not suppress the newly
                // imported stores on other nodes during sync.
                if (!backend_.tombstones_.empty()) {
                    backend_.tombstones_.clear();
                    try { backend_.replicate_tombstones(); } catch (...) {}
                }
                flag_.store(false);
                // Sync from cluster to pick up the replicated data on this node
                try { backend_.sync_from_cluster(); } catch (...) {}
            } else {
                flag_.store(false);
            }
        }
        void feed(const char* data, size_t len) override { inner_->feed(data, len); }
        bool done() const override { return inner_->done(); }
        bool ok() const override { return inner_->ok(); }
        std::string error_message() const override { return inner_->error_message(); }
        std::vector<std::function<void()>> take_deferred() override { return inner_->take_deferred(); }
        Progress progress() const override { return inner_->progress(); }
    private:
        std::unique_ptr<ImportSession> inner_;
        std::atomic<bool>& flag_;
        ClusterMediaBackend& backend_;
    };

    return std::make_unique<ImportGuardSession>(std::move(session), importing_, *this);
}

void ClusterMediaBackend::vacuum_store(const std::string& store_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    local_.vacuum_store(store_id);
    replicate_store(store_id);
    replicate_index();
    cluster_.vacuum_all_nodes();
}

void ClusterMediaBackend::vacuum_all_stores() {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    local_.vacuum_all_stores();
    for (const auto& sid : local_.store_ids()) {
        replicate_store(sid);
    }
    replicate_index();

    // Orphan cleanup: remove cluster groups not referenced by current data.
    // SAFETY: Skip orphan cleanup if the local database appears empty —
    // an empty local DB would cause ALL cluster groups to be treated as
    // orphans, deleting the entire dataset across all nodes.
    auto local_media = local_.media_ids();
    auto local_stores = local_.store_ids();
    if (!local_media.empty() || !local_stores.empty()) {
        std::unordered_set<uint64_t> known_gids;
        known_gids.insert(cluster_group_id("index"));
        known_gids.insert(cluster_group_id("tombstones"));
        for (const auto& sid : local_stores) {
            known_gids.insert(cluster_group_id("store:" + sid));
        }
        for (const auto& mid : local_media) {
            known_gids.insert(cluster_group_id("media:" + mid));
        }

        std::unordered_set<uint64_t> all_gids;
        for (const auto& pg : cluster_.list_peer_groups()) {
            for (auto gid : pg.groups)
                all_gids.insert(gid);
        }

        int orphans = 0;
        auto& cli = cluster_.getClient();
        for (uint64_t gid : all_gids) {
            if (known_gids.count(gid)) continue;
            std::cerr << "[VACUUM] removing orphan group " << gid << "\n";
            try {
                if (cli) cli->remove(gid);
                ++orphans;
            } catch (...) {}
        }
        if (orphans > 0)
            std::cerr << "[VACUUM] removed " << orphans << " orphan groups\n";
    } else {
        std::cerr << "[VACUUM] skipping orphan cleanup: local database is empty\n";
    }

    cguard.unlock();
    cluster_.vacuum_all_nodes();
}

// ---- cluster sync ----
void ClusterMediaBackend::sync_from_cluster() {
    // Route all sync requests through the scheduler so sync/repair runs
    // serialize behind one worker instead of interleaving across threads.
    if (sync_running_.load()) {
        request_sync(true);
        return;
    }

    // Before scheduler startup (initial boot), run sync inline.
    sync_from_cluster_now();
}

void ClusterMediaBackend::sync_from_cluster_now() {
    if (!cluster_.isRunning()) {
        std::cerr << "[CLUSTER-SYNC] cluster not running, skipping sync\n";
        return;
    }
    if (cluster_.isImportRunning()) {
        std::cerr << "[CLUSTER-SYNC] global import flag active, skipping sync\n";
        return;
    }
    if (importing_.load()) {
        std::cerr << "[CLUSTER-SYNC] import in progress, skipping sync\n";
        return;
    }

    auto import_active = [this]() { return this->import_active(); };

    if (import_active()) {
        std::cerr << "[CLUSTER-SYNC] import active, aborting before index scan\n";
        return;
    }

    // Fetch index from cluster (contains store/album/ACL metadata, no media data)
    std::vector<uint8_t> index_data;
    bool fetch_ok = cluster_.fetch("index", index_data);
    if (import_active()) {
        std::cerr << "[CLUSTER-SYNC] import became active during index fetch, aborting sync\n";
        return;
    }
    if (!fetch_ok || index_data.empty()) {
        if (import_active()) {
            std::cerr << "[CLUSTER-SYNC] import active, skipping index fetch retry\n";
            return;
        }
        // Retry once after warmup — connections may be stale
        cluster_.warmup_read_clients();
        std::this_thread::sleep_for(std::chrono::milliseconds(500));
        fetch_ok = cluster_.fetch("index", index_data);
        if (import_active()) {
            std::cerr << "[CLUSTER-SYNC] import became active during index fetch retry, aborting sync\n";
            return;
        }
    }
    if (!fetch_ok || index_data.empty()) {
        if (import_active()) {
            std::cerr << "[CLUSTER-SYNC] import active after index fetch failure, suppressing sync error\n";
            return;
        }
        if (fetch_ok) {
            // fetch succeeded but returned empty data — fresh/empty cluster
            std::cerr << "[CLUSTER-SYNC] index returned empty data — fresh/empty cluster\n";
            initial_sync_ok_.store(true);
        } else {
            std::cerr << "[CLUSTER-SYNC] index fetch failed (exception in cluster layer)\n";
        }
        return;
    }

    // Sync may have started just before import began. Re-check before mutating
    // local state so tombstones/index metadata cannot interleave with import.
    if (import_active()) {
        std::cerr << "[CLUSTER-SYNC] import became active mid-sync, aborting before index apply\n";
        return;
    }

    std::cerr << "[CLUSTER-SYNC] index fetched OK, " << index_data.size() << " bytes\n";
    {
        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        local_.load_index_from_buffer(index_data);
    }
    initial_sync_ok_.store(true);

    // Load tombstones from cluster and delete any stores that were deleted elsewhere
    load_tombstones();

    if (import_active()) {
        std::cerr << "[CLUSTER-SYNC] import became active mid-sync, aborting before tombstones\n";
        return;
    }

    {
        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        apply_tombstones();
    }

    // Snapshot store IDs under shared lock
    std::vector<std::string> sids;
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        sids = local_.store_ids();
    }

    // Fetch each store metadata (album/media records) from cluster.
    // Uses load_store_metadata_from_buffer which loads media metadata
    // but NOT raw media data — keeps RAM usage low.
    std::cerr << "[CLUSTER-SYNC] fetching metadata for " << sids.size() << " stores\n";
    last_sync_stores_complete_.store(true);
    for (const auto& sid : sids) {
        if (import_active()) {
            std::cerr << "[CLUSTER-SYNC] import became active mid-sync, aborting store metadata fetch\n";
            return;
        }

        std::vector<uint8_t> store_data;
        bool ok = cluster_.fetch("store:" + sid, store_data);
        if (!ok || store_data.empty()) {
            // Retry once — peer connections may be stale
            std::this_thread::sleep_for(std::chrono::milliseconds(300));
            ok = cluster_.fetch("store:" + sid, store_data);
        }
        if (!ok || store_data.empty()) {
            std::cerr << "[CLUSTER-SYNC] store:" << sid
                      << (ok ? " returned empty data" : " fetch failed") << "\n";
            last_sync_stores_complete_.store(false);
            continue;
        }
        {
            std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
            local_.load_store_metadata_from_buffer(sid, store_data);
        }
    }
}

BinDb& ClusterMediaBackend::local() { return local_; }

void ClusterMediaBackend::repair_index() {
    if (!cluster_.isRunning()) {
        std::cerr << "[CLUSTER-REPAIR-INDEX] cluster not running\n";
        return;
    }

    // Step 1: Collect all unique group IDs from every peer
    auto peer_groups = cluster_.list_peer_groups();
    std::unordered_set<uint64_t> all_gids;
    for (const auto& pg : peer_groups) {
        if (!pg.online) continue;
        all_gids.insert(pg.groups.begin(), pg.groups.end());
    }

    std::cerr << "[CLUSTER-REPAIR-INDEX] scanning " << all_gids.size()
              << " unique group IDs across peers\n";

    // Step 2: Try to retrieve every group ID, check if it's store metadata
    auto& cli = cluster_.getClient();
    if (!cli) {
        std::cerr << "[CLUSTER-REPAIR-INDEX] no parity client available\n";
        return;
    }

    // Known magic bytes for store metadata
    static constexpr char S1[4] = {'M', 'D', 'S', '1'};
    static constexpr char S2[4] = {'M', 'D', 'S', '2'};
    static constexpr char S3[4] = {'M', 'D', 'S', '3'};
    // Known magic bytes for index
    static constexpr char I2[4] = {'M', 'D', 'B', '2'};
    static constexpr char I3[4] = {'M', 'D', 'B', '3'};
    static constexpr char I4[4] = {'M', 'D', 'B', '4'};

    auto is_store_magic = [&](const std::vector<uint8_t>& data) {
        if (data.size() < 4) return false;
        return std::memcmp(data.data(), S1, 4) == 0
            || std::memcmp(data.data(), S2, 4) == 0
            || std::memcmp(data.data(), S3, 4) == 0;
    };

    auto is_index_magic = [&](const std::vector<uint8_t>& data) {
        if (data.size() < 4) return false;
        return std::memcmp(data.data(), I2, 4) == 0
            || std::memcmp(data.data(), I3, 4) == 0
            || std::memcmp(data.data(), I4, 4) == 0;
    };

    // Skip group IDs we already know (index, stores, media)
    uint64_t index_gid = cluster_group_id("index");

    std::unordered_set<uint64_t> known_gids;
    known_gids.insert(index_gid);
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        for (const auto& sid : local_.store_ids()) {
            known_gids.insert(cluster_group_id("store:" + sid));
        }
        for (const auto& mid : local_.media_ids()) {
            known_gids.insert(cluster_group_id("media:" + mid));
        }
    }

    // First, try to load the index from the cluster (may contain store IDs we don't know locally)
    std::vector<uint8_t> index_data;
    bool index_ok = cluster_.fetch("index", index_data);
    if (index_ok && !index_data.empty() && is_index_magic(index_data)) {
        std::cerr << "[CLUSTER-REPAIR-INDEX] found valid index (" << index_data.size() << " bytes), loading\n";
        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        local_.load_index_from_buffer(index_data);
    }

    // Now try unknown group IDs — these might be orphaned stores
    int found_stores = 0;
    int scanned = 0;
    for (uint64_t gid : all_gids) {
        if (known_gids.count(gid)) continue;
        ++scanned;

        std::vector<uint8_t> data;
        try {
            data = cli->retrieve(gid);
        } catch (...) {
            continue;
        }
        if (data.empty()) continue;

        if (is_store_magic(data)) {
            // Found orphaned store metadata! We need to figure out its store_id.
            // Parse the store buffer to extract album IDs, then find a store_id
            // from the album records.
            std::cerr << "[CLUSTER-REPAIR-INDEX] found store metadata at gid=" << gid
                      << " (" << data.size() << " bytes)\n";

            // We need to find the store_id that hashes to this gid.
            // Check all locally known store IDs first.
            std::string matched_sid;
            {
                std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
                for (const auto& sid : local_.store_ids()) {
                    if (cluster_group_id("store:" + sid) == gid) {
                        matched_sid = sid;
                        break;
                    }
                }
            }

            if (!matched_sid.empty()) {
                std::cerr << "[CLUSTER-REPAIR-INDEX] matched gid=" << gid
                          << " to known store:" << matched_sid << "\n";
                std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
                local_.load_store_metadata_from_buffer(matched_sid, data);
                ++found_stores;
            } else {
                std::cerr << "[CLUSTER-REPAIR-INDEX] store at gid=" << gid
                          << " has no matching local store_id (orphan)\n";
            }
        } else if (is_index_magic(data)) {
            std::cerr << "[CLUSTER-REPAIR-INDEX] found additional index at gid=" << gid
                      << " (" << data.size() << " bytes), loading\n";
            std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
            local_.load_index_from_buffer(data);
        }
    }

    // Step 3: With the (possibly updated) index, now try to fetch ALL store metadata
    std::vector<std::string> sids;
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        sids = local_.store_ids();
    }

    int repaired = 0;
    for (const auto& sid : sids) {
        std::vector<uint8_t> store_data;
        bool ok = cluster_.fetch("store:" + sid, store_data);
        if (!ok || store_data.empty()) {
            std::cerr << "[CLUSTER-REPAIR-INDEX] store:" << sid << " fetch failed\n";
            continue;
        }
        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        local_.load_store_metadata_from_buffer(sid, store_data);
        ++repaired;
    }

    // Step 4: Re-replicate the repaired index
    {
        std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
        auto sids2 = local_.store_ids();
        std::size_t total_albums = 0;
        std::size_t total_media = 0;
        for (const auto& sid : sids2) {
            auto albums = local_.list_albums(sid);
            total_albums += albums.size();
            for (const auto& a : albums)
                total_media += local_.list_media(a.id).size();
        }
        std::cerr << "[CLUSTER-REPAIR-INDEX] result: " << sids2.size() << " stores, "
                  << total_albums << " albums, " << total_media << " media items"
                  << " (scanned=" << scanned << " unknown gids, found_stores=" << found_stores
                  << ", repaired=" << repaired << ")\n";
    }

    initial_sync_ok_.store(true);
    replicate_index();
}

void ClusterMediaBackend::replicate_index(bool force) {
    if (!cluster_.isRunning()) return;
    auto buf = local_.save_index_to_buffer();
    if (buf.empty()) return;

    if (!force) {
        // Safety: never push an empty index (would erase data on other nodes).
        auto local_count = local_.store_ids().size();
        if (local_count == 0) return;
    }

    cluster_.replicate("index", buf.data(), buf.size());
}

void ClusterMediaBackend::replicate_store(const std::string& store_id) {
    if (!cluster_.isRunning()) return;
    // Replicate store metadata only (media data stored as separate keys)
    auto buf = local_.save_store_metadata_to_buffer(store_id);
    if (buf.empty()) {
        return;
    }
    cluster_.replicate("store:" + store_id, buf.data(), buf.size());
}

void ClusterMediaBackend::replicate_tombstones() {
    if (!cluster_.isRunning()) return;
    // Serialise: u32 count, then each id as length-prefixed string.
    // Always replicate — even when empty — so other nodes see the cleared list.
    std::vector<std::uint8_t> buf;
    auto push_u32 = [&](std::uint32_t v) {
        buf.push_back(static_cast<uint8_t>(v & 0xFF));
        buf.push_back(static_cast<uint8_t>((v >> 8) & 0xFF));
        buf.push_back(static_cast<uint8_t>((v >> 16) & 0xFF));
        buf.push_back(static_cast<uint8_t>((v >> 24) & 0xFF));
    };
    push_u32(static_cast<std::uint32_t>(tombstones_.size()));
    for (const auto& id : tombstones_) {
        push_u32(static_cast<std::uint32_t>(id.size()));
        buf.insert(buf.end(), id.begin(), id.end());
    }
    cluster_.replicate("tombstones", buf.data(), buf.size());
}

void ClusterMediaBackend::load_tombstones() {
    if (!cluster_.isRunning()) return;
    try {
        std::vector<uint8_t> buf;
        bool ok = cluster_.fetch("tombstones", buf);
        if (!ok || buf.size() < 4) return;
        size_t pos = 0;
        auto read_u32 = [&]() -> std::uint32_t {
            if (pos + 4 > buf.size()) return 0;
            std::uint32_t v = static_cast<uint8_t>(buf[pos])
                            | (static_cast<uint8_t>(buf[pos+1]) << 8)
                            | (static_cast<uint8_t>(buf[pos+2]) << 16)
                            | (static_cast<uint8_t>(buf[pos+3]) << 24);
            pos += 4;
            return v;
        };
        // REPLACE local tombstones with cluster tombstones (not merge).
        // Merging caused a loop: deleted tombstones were re-inserted from
        // the cluster on every sync, permanently deleting stores that
        // a successful import had already cleared.
        std::set<std::string> cluster_tombstones;
        auto count = read_u32();
        for (std::uint32_t i = 0; i < count && pos < buf.size(); ++i) {
            auto len = read_u32();
            if (pos + len > buf.size()) break;
            std::string id(buf.begin() + pos, buf.begin() + pos + len);
            pos += len;
            cluster_tombstones.insert(std::move(id));
        }
        tombstones_ = std::move(cluster_tombstones);
    } catch (...) {
        // No tombstones blob yet — that's fine
    }
}

void ClusterMediaBackend::apply_tombstones() {
    if (importing_.load() || cluster_.isImportRunning()) {
        std::cerr << "[CLUSTER-SYNC] apply_tombstones skipped: import active\n";
        return;
    }

    // Remove any stores present in the tombstone list (local only).
    // Do NOT call cluster_.remove() here — that would delete data from the
    // entire cluster. Only the node that initiated the delete (delete_store)
    // should remove cluster blobs. Other nodes only suppress local visibility.
    if (!tombstones_.empty())
        std::cerr << "[CLUSTER-SYNC] apply_tombstones: " << tombstones_.size() << " tombstones\n";
    for (const auto& id : tombstones_) {
        if (local_.get_store(id)) {
            std::cerr << "[CLUSTER-SYNC] TOMBSTONE DELETING store " << id << " (local only)\n";
            local_.delete_store(id);
        }
    }
}

void ClusterMediaBackend::repair_replication() {
    if (!cluster_.isRunning()) return;
    if (importing_.load() || cluster_.isImportRunning()) return;
    size_t total_peers = cluster_.getTotalPeers();
    if (total_peers < 2) return;

    const auto& cfg = cluster_.getConfig();
    size_t required_shards = cfg.data_blocks + cfg.parity_blocks;

    auto peer_groups = cluster_.list_peer_groups();
    std::vector<std::unordered_set<uint64_t>> node_group_sets(peer_groups.size());
    for (size_t i = 0; i < peer_groups.size(); ++i) {
        node_group_sets[i].insert(peer_groups[i].groups.begin(),
                                  peer_groups[i].groups.end());
    }

    size_t online_count = 0;
    for (const auto& pg : peer_groups)
        if (pg.online) ++online_count;
    if (online_count < required_shards) return; // not enough nodes online to repair

    // Always check index replication — even before initial_sync_ok_.
    // But ONLY push if this node actually has stores. An empty index
    // must never overwrite a non-empty one in the cluster.
    uint64_t index_gid = cluster_group_id("index");
    int index_shards = 0;
    for (size_t i = 0; i < peer_groups.size(); ++i) {
        if (peer_groups[i].online && node_group_sets[i].count(index_gid))
            ++index_shards;
    }
    if (index_shards < static_cast<int>(required_shards)) {
        auto sids = local_.store_ids();
        if (!sids.empty()) {
            auto buf = local_.save_index_to_buffer();
            if (!buf.empty()) {
                std::cerr << "[CLUSTER-REPAIR] re-replicating index ("
                          << index_shards << "/" << required_shards
                          << ", " << sids.size() << " local stores)\n";
                cluster_.replicate("index", buf.data(), buf.size());
            }
        }
    }

    // Everything below requires a successful initial sync
    if (!initial_sync_ok_.load()) return;

    // Check stores
    for (const auto& sid : local_.store_ids()) {
        uint64_t gid = cluster_group_id("store:" + sid);
        int shards = 0;
        for (size_t i = 0; i < peer_groups.size(); ++i) {
            if (peer_groups[i].online && node_group_sets[i].count(gid))
                ++shards;
        }
        if (shards < static_cast<int>(required_shards)) {
            DBG_LOG("[CLUSTER-REPAIR] re-replicating store:" << sid
                      << " (" << shards << "/" << required_shards << ")\n");
            replicate_store(sid);
        }
    }

    // Check media keys
    for (const auto& mid : local_.media_ids()) {
        uint64_t gid = cluster_group_id("media:" + mid);
        int shards = 0;
        for (size_t i = 0; i < peer_groups.size(); ++i) {
            if (peer_groups[i].online && node_group_sets[i].count(gid))
                ++shards;
        }
        if (shards < static_cast<int>(required_shards)) {
            DBG_LOG("[CLUSTER-REPAIR] re-replicating media:" << mid
                      << " (" << shards << "/" << required_shards << ")\n");
            // Fetch data from cluster and re-replicate
            std::vector<uint8_t> data;
            cluster_.fetch("media:" + mid, data);
            if (!data.empty()) {
                cluster_.replicate("media:" + mid, data.data(), data.size());
            }
        }
    }
}

void ClusterMediaBackend::sync_loop() {
    int cycle = 0;
    auto next_sync = std::chrono::steady_clock::now()
                   + std::chrono::seconds(sync_interval_seconds());

    while (sync_running_.load()) {
        bool do_sync = false;
        bool do_repair = false;
        std::uint64_t done_ticket = 0;

        {
            std::unique_lock<std::mutex> lock(sync_mutex_);
            sync_cv_.wait_until(lock, next_sync, [this] {
                return !sync_running_.load() || sync_requested_ || repair_requested_;
            });
            if (!sync_running_.load()) break;

            const auto now = std::chrono::steady_clock::now();
            if (now >= next_sync) {
                sync_requested_ = true;
                if (++cycle >= 10) {
                    cycle = 0;
                    repair_requested_ = true;
                }
                next_sync = now + std::chrono::seconds(sync_interval_seconds());
            }

            if (import_active()) {
                // Keep queued jobs pending while import is active.
                continue;
            }

            if (sync_requested_) {
                sync_requested_ = false;
                do_sync = true;
                done_ticket = sync_ticket_;
            } else if (repair_requested_) {
                repair_requested_ = false;
                do_repair = true;
            }
        }

        try {
            if (do_sync) {
                sync_from_cluster_now();
            } else if (do_repair) {
                repair_replication();
            }
        } catch (const std::exception& e) {
            std::cerr << "[CLUSTER-SYNC] scheduled task failed: " << e.what() << "\n";
        } catch (...) {
            std::cerr << "[CLUSTER-SYNC] scheduled task failed (unknown)\n";
        }

        if (do_sync) {
            std::lock_guard<std::mutex> lock(sync_mutex_);
            if (sync_done_ticket_ < done_ticket) sync_done_ticket_ = done_ticket;
            sync_cv_.notify_all();
        }
    }
}

} // namespace mediadb
