#include "backend.h"
#include "cluster.h"

#include <algorithm>
#include <iostream>
#include <sstream>
#include <unordered_set>

namespace mediadb {

// ---- BinDb ----

BinDb::BinDb(const std::string& db_dir) : db_dir_(db_dir), memory_only_(false) {
    fs::create_directories(db_dir_);
    load();
}

BinDb::BinDb() : memory_only_(true) {}

fs::path BinDb::index_file() const { return fs::path(db_dir_) / "index.mdb"; }
fs::path BinDb::store_file(const std::string& store_id) const { return fs::path(db_dir_) / (store_id + ".mdb"); }

std::string BinDb::detect_kind(std::string ext) {
    std::transform(ext.begin(), ext.end(), ext.begin(),
                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
    if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" || ext == ".bmp" || ext == ".gif" || ext == ".avif") return "image";
    if (ext == ".mp4" || ext == ".mov" || ext == ".mkv" || ext == ".webm" || ext == ".avi" || ext == ".m4v" || ext == ".ts" || ext == ".flv") return "video";
    if (ext == ".mp3" || ext == ".flac" || ext == ".wav" || ext == ".ogg" || ext == ".aac" || ext == ".m4a" || ext == ".opus") return "audio";
    return "unknown";
}

// ---- stores ----

StoreRecord BinDb::create_store(const std::string& name) {
    StoreRecord s;
    {
        std::unique_lock lock(mutex_);
        s.id = make_uuid();
        s.name = name;
        s.created_at = now_iso8601();
        stores_[s.id] = s;
        save_index_locked();
        ensure_store_file_locked(s.id);
    }
    return s;
}

std::optional<StoreRecord> BinDb::get_store(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(id);
    if (it == stores_.end()) return std::nullopt;
    return it->second;
}

std::vector<StoreRecord> BinDb::list_stores() const {
    std::shared_lock lock(mutex_);
    std::vector<StoreRecord> out;
    out.reserve(stores_.size());
    for (const auto& [k, v] : stores_) out.push_back(v);
    return out;
}

bool BinDb::delete_store(const std::string& id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(id);
        if (it == stores_.end()) return false;
        for (const auto& aid : it->second.album_ids) {
            auto ait = albums_.find(aid);
            if (ait != albums_.end()) {
                for (const auto& mid : ait->second.media_ids) media_.erase(mid);
                albums_.erase(ait);
            }
        }
        mmaps_.erase(id);
        if (!memory_only_) {
            auto sf = store_file(id);
            if (fs::exists(sf)) fs::remove(sf);
        }
        stores_.erase(it);
        if (!memory_only_) save_index_locked();
    }
    return true;
}

bool BinDb::rename_store(const std::string& id, const std::string& new_name) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(id);
        if (it == stores_.end()) return false;
        it->second.name = new_name;
        save_index_locked();
    }
    return true;
}

// ---- store auth domains ----

std::string BinDb::add_store_domain(const std::string& store_id, AuthDomain domain) {
    std::string did;
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return {};
        domain.id = make_uuid();
        did = domain.id;
        it->second.auth_domains.push_back(std::move(domain));
        save_index_locked();
    }
    return did;
}

bool BinDb::remove_store_domain(const std::string& store_id, const std::string& domain_id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return false;
        auto& domains = it->second.auth_domains;
        auto dit = std::find_if(domains.begin(), domains.end(),
                                [&](const AuthDomain& d) { return d.id == domain_id; });
        if (dit == domains.end()) return false;
        domains.erase(dit);
        save_index_locked();
    }
    return true;
}

std::vector<AuthDomain> BinDb::list_store_domains(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(store_id);
    if (it == stores_.end()) return {};
    return it->second.auth_domains;
}

std::optional<AuthDomain> BinDb::find_domain_by_label(const std::string& label) const {
    std::shared_lock lock(mutex_);
    for (const auto& [sid, store] : stores_) {
        for (const auto& d : store.auth_domains) {
            if (d.label == label) return d;
        }
    }
    return std::nullopt;
}

// ---- store ACLs ----

std::string BinDb::add_store_acl(const std::string& store_id, StoreACL acl) {
    std::string aid;
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return {};
        acl.id = make_uuid();
        aid = acl.id;
        it->second.acls.push_back(std::move(acl));
        save_index_locked();
    }
    return aid;
}

bool BinDb::remove_store_acl(const std::string& store_id, const std::string& acl_id) {
    {
        std::unique_lock lock(mutex_);
        auto it = stores_.find(store_id);
        if (it == stores_.end()) return false;
        auto& acls = it->second.acls;
        auto ait = std::find_if(acls.begin(), acls.end(),
                                [&](const StoreACL& a) { return a.id == acl_id; });
        if (ait == acls.end()) return false;
        acls.erase(ait);
        save_index_locked();
    }
    return true;
}

std::vector<StoreACL> BinDb::list_store_acls(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    auto it = stores_.find(store_id);
    if (it == stores_.end()) return {};
    return it->second.acls;
}

std::vector<std::string> BinDb::all_acl_group_ids() const {
    std::shared_lock lock(mutex_);
    std::set<std::string> ids;
    for (const auto& [sid, s] : stores_) {
        for (const auto& acl : s.acls) {
            ids.insert(acl.group_id);
        }
    }
    return {ids.begin(), ids.end()};
}

// ---- albums ----

std::optional<AlbumRecord> BinDb::create_album(const std::string& store_id, const std::string& name) {
    std::optional<AlbumRecord> result;
    {
        std::unique_lock lock(mutex_);
        auto sit = stores_.find(store_id);
        if (sit == stores_.end()) return std::nullopt;

        AlbumRecord a;
        a.id = make_uuid();
        a.name = name;
        a.store_id = store_id;
        a.created_at = now_iso8601();
        albums_[a.id] = a;
        sit->second.album_ids.push_back(a.id);
        append_album_locked(store_id, albums_[a.id]);
        result = albums_[a.id];
    }
    return result;
}

std::optional<AlbumRecord> BinDb::get_album(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = albums_.find(id);
    if (it == albums_.end()) return std::nullopt;
    return it->second;
}

std::vector<AlbumRecord> BinDb::list_albums(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    std::vector<AlbumRecord> out;
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return out;
    for (const auto& aid : sit->second.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) out.push_back(ait->second);
    }
    return out;
}

bool BinDb::delete_album(const std::string& id) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto ait = albums_.find(id);
        if (ait == albums_.end()) return false;
        store_id = ait->second.store_id;
        auto sit = stores_.find(store_id);
        if (sit != stores_.end()) {
            auto& aids = sit->second.album_ids;
            aids.erase(std::remove(aids.begin(), aids.end(), id), aids.end());
        }
        // Zero all media records belonging to this album
        for (const auto& mid : ait->second.media_ids) {
            auto mit2 = media_.find(mid);
            if (mit2 != media_.end()) {
                zero_record_locked(store_id, mit2->second.record_offset);
                media_.erase(mit2);
            }
        }
        // Zero the album record itself
        zero_record_locked(store_id, ait->second.record_offset);
        albums_.erase(ait);
    }
    return true;
}

bool BinDb::set_album_public(const std::string& album_id, bool is_public) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto it = albums_.find(album_id);
        if (it == albums_.end()) return false;
        // Zero old album record, update flag, append new one
        zero_record_locked(it->second.store_id, it->second.record_offset);
        it->second.is_public = is_public;
        store_id = it->second.store_id;
        append_album_locked(store_id, it->second);
    }
    return true;
}

bool BinDb::is_album_public(const std::string& album_id) const {
    std::shared_lock lock(mutex_);
    auto it = albums_.find(album_id);
    if (it == albums_.end()) return false;
    return it->second.is_public;
}

bool BinDb::is_media_public(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return false;
    auto ait = albums_.find(it->second.album_id);
    if (ait == albums_.end()) return false;
    return ait->second.is_public;
}

std::vector<AlbumRecord> BinDb::list_public_albums() const {
    std::shared_lock lock(mutex_);
    std::vector<AlbumRecord> out;
    for (const auto& [id, album] : albums_) {
        if (album.is_public) out.push_back(album);
    }
    return out;
}

// ---- media ----

std::optional<MediaRecord> BinDb::add_media(const std::string& album_id,
                                             const std::string& filename,
                                             const std::string& content_type,
                                             std::vector<std::uint8_t> data) {
    std::optional<MediaRecord> result;
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto ait = albums_.find(album_id);
        if (ait == albums_.end()) return std::nullopt;

        store_id = ait->second.store_id;
        const fs::path p(filename);
        MediaRecord m;
        m.id = make_uuid();
        m.original_filename = filename;
        m.media_kind = detect_kind(p.extension().string());
        m.content_type = content_type;
        m.album_id = album_id;
        m.store_id = store_id;
        m.created_at = now_iso8601();
        m.size_bytes = data.size();
        m.pending_data = std::move(data);
        m.data_offset = 0;
        const std::string mid = m.id;
        media_[mid] = std::move(m);
        ait->second.media_ids.push_back(mid);

        append_media_locked(store_id, media_[mid]);
        result = media_[mid];
    }
    return result;
}

std::optional<MediaRecord> BinDb::get_media(const std::string& id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(id);
    if (it == media_.end()) return std::nullopt;
    return it->second;
}

std::vector<MediaRecord> BinDb::list_media(const std::string& album_id) const {
    std::shared_lock lock(mutex_);
    std::vector<MediaRecord> out;
    auto ait = albums_.find(album_id);
    if (ait == albums_.end()) return out;
    for (const auto& mid : ait->second.media_ids) {
        auto mit = media_.find(mid);
        if (mit != media_.end()) out.push_back(mit->second);
    }
    return out;
}

bool BinDb::delete_media(const std::string& id) {
    std::string store_id;
    {
        std::unique_lock lock(mutex_);
        auto mit = media_.find(id);
        if (mit == media_.end()) return false;
        store_id = mit->second.store_id;
        auto rec_off = mit->second.record_offset;
        auto ait = albums_.find(mit->second.album_id);
        if (ait != albums_.end()) {
            auto& mids = ait->second.media_ids;
            mids.erase(std::remove(mids.begin(), mids.end(), id), mids.end());
        }
        media_.erase(mit);
        zero_record_locked(store_id, rec_off);
    }
    return true;
}

// ---- read raw media data ----

std::vector<std::uint8_t> BinDb::read_media_data(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return {};
    const auto& m = it->second;
    if (m.size_bytes == 0) return {};
    if (!m.pending_data.empty()) return m.pending_data;

    auto mit = mmaps_.find(m.store_id);
    if (mit != mmaps_.end() && mit->second.valid()) {
        const auto& mm = mit->second;
        if (m.data_offset + m.size_bytes <= mm.size) {
            const auto* ptr = mm.data() + m.data_offset;
            return std::vector<std::uint8_t>(ptr, ptr + m.size_bytes);
        }
    }
    std::ifstream in(store_file(m.store_id), std::ios::binary);
    if (!in.is_open()) return {};
    in.seekg(static_cast<std::streamoff>(m.data_offset));
    std::vector<std::uint8_t> buf(m.size_bytes);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(m.size_bytes));
    return buf;
}

std::vector<std::uint8_t> BinDb::read_media_data_range(const std::string& media_id,
                                                         std::uint64_t offset,
                                                         std::uint64_t length) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return {};
    const auto& m = it->second;
    if (m.size_bytes == 0 || offset >= m.size_bytes) return {};
    const std::uint64_t avail = m.size_bytes - offset;
    const std::uint64_t to_read = std::min(length, avail);

    if (!m.pending_data.empty()) {
        return std::vector<std::uint8_t>(
            m.pending_data.begin() + static_cast<std::ptrdiff_t>(offset),
            m.pending_data.begin() + static_cast<std::ptrdiff_t>(offset + to_read));
    }

    auto mit = mmaps_.find(m.store_id);
    if (mit != mmaps_.end() && mit->second.valid()) {
        const auto& mm = mit->second;
        const auto abs_off = m.data_offset + offset;
        if (abs_off + to_read <= mm.size) {
            const auto* ptr = mm.data() + abs_off;
            return std::vector<std::uint8_t>(ptr, ptr + to_read);
        }
    }
    std::ifstream in(store_file(m.store_id), std::ios::binary);
    if (!in.is_open()) return {};
    in.seekg(static_cast<std::streamoff>(m.data_offset + offset));
    std::vector<std::uint8_t> buf(to_read);
    in.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(to_read));
    return buf;
}

std::uint64_t BinDb::get_media_size(const std::string& media_id) const {
    std::shared_lock lock(mutex_);
    auto it = media_.find(media_id);
    if (it == media_.end()) return 0;
    return it->second.size_bytes;
}

// ---- export ----

bool BinDb::export_db(const std::string& path) const {
    std::shared_lock lock(mutex_);
    const std::string tmp = path + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return false;
        out.write(EXPORT_MAGIC_V2, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(stores_.size()));

        for (const auto& [sid, store] : stores_) {
            bin_write_str(out, store.id);
            bin_write_str(out, store.name);
            bin_write_str(out, store.created_at);
            bin_write_u32(out, static_cast<std::uint32_t>(store.album_ids.size()));

            for (const auto& aid : store.album_ids) {
                auto ait = albums_.find(aid);
                if (ait == albums_.end()) {
                    bin_write_str(out, aid);
                    bin_write_str(out, "");
                    bin_write_str(out, "");
                    { std::uint8_t pub = 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
                    bin_write_u32(out, 0);
                    continue;
                }
                const auto& album = ait->second;
                bin_write_str(out, album.id);
                bin_write_str(out, album.name);
                bin_write_str(out, album.created_at);
                { std::uint8_t pub = album.is_public ? 1 : 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
                bin_write_u32(out, static_cast<std::uint32_t>(album.media_ids.size()));

                for (const auto& mid : album.media_ids) {
                    auto mit = media_.find(mid);
                    if (mit == media_.end()) continue;
                    const auto& m = mit->second;

                    bin_write_str(out, m.id);
                    bin_write_str(out, m.original_filename);
                    bin_write_str(out, m.media_kind);
                    bin_write_str(out, m.content_type);
                    bin_write_str(out, m.created_at);
                    bin_write_u64(out, m.size_bytes);

                    if (!m.pending_data.empty()) {
                        out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                                  static_cast<std::streamsize>(m.pending_data.size()));
                    } else if (m.data_offset > 0 && m.size_bytes > 0) {
                        std::ifstream sin(store_file(sid), std::ios::binary);
                        if (sin.is_open()) {
                            sin.seekg(static_cast<std::streamoff>(m.data_offset));
                            constexpr std::size_t BUF = 1024 * 1024;
                            std::vector<char> buf(BUF);
                            std::uint64_t remaining = m.size_bytes;
                            while (remaining > 0 && sin.good()) {
                                auto chunk = static_cast<std::streamsize>(
                                    std::min<std::uint64_t>(remaining, BUF));
                                sin.read(buf.data(), chunk);
                                auto got = sin.gcount();
                                if (got > 0) out.write(buf.data(), got);
                                remaining -= static_cast<std::uint64_t>(got);
                                if (got < chunk) break;
                            }
                        }
                    }
                }
            }
        }
    }
    fs::rename(tmp, path);
    return true;
}

std::vector<std::uint8_t> BinDb::export_db_to_buffer() const {
    std::shared_lock lock(mutex_);
    std::ostringstream oss(std::ios::binary);
    oss.write(EXPORT_MAGIC_V2, 4);

    auto wr_u32 = [&](std::uint32_t v) { oss.write(reinterpret_cast<const char*>(&v), 4); };
    auto wr_u64 = [&](std::uint64_t v) { oss.write(reinterpret_cast<const char*>(&v), 8); };
    auto wr_str = [&](const std::string& s) {
        auto len = static_cast<std::uint32_t>(s.size());
        wr_u32(len);
        if (len > 0) oss.write(s.data(), len);
    };

    wr_u32(static_cast<std::uint32_t>(stores_.size()));

    for (const auto& [sid, store] : stores_) {
        wr_str(store.id);
        wr_str(store.name);
        wr_str(store.created_at);
        wr_u32(static_cast<std::uint32_t>(store.album_ids.size()));

        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) {
                wr_str(aid);
                wr_str("");
                wr_str("");
                { std::uint8_t pub = 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
                wr_u32(0);
                continue;
            }
            const auto& album = ait->second;
            wr_str(album.id);
            wr_str(album.name);
            wr_str(album.created_at);
            { std::uint8_t pub = album.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
            wr_u32(static_cast<std::uint32_t>(album.media_ids.size()));

            for (const auto& mid : album.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                const auto& m = mit->second;

                wr_str(m.id);
                wr_str(m.original_filename);
                wr_str(m.media_kind);
                wr_str(m.content_type);
                wr_str(m.created_at);
                wr_u64(m.size_bytes);

                if (!m.pending_data.empty()) {
                    oss.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                } else if (m.data_offset > 0 && m.size_bytes > 0) {
                    std::ifstream sin(store_file(sid), std::ios::binary);
                    if (sin.is_open()) {
                        sin.seekg(static_cast<std::streamoff>(m.data_offset));
                        constexpr std::size_t BUF = 1024 * 1024;
                        std::vector<char> buf(BUF);
                        std::uint64_t remaining = m.size_bytes;
                        while (remaining > 0 && sin.good()) {
                            auto chunk = static_cast<std::streamsize>(
                                std::min<std::uint64_t>(remaining, BUF));
                            sin.read(buf.data(), chunk);
                            auto got = sin.gcount();
                            if (got > 0) oss.write(buf.data(), got);
                            remaining -= static_cast<std::uint64_t>(got);
                            if (got < chunk) break;
                        }
                    }
                }
            }
        }
    }

    std::string s = std::move(oss).str();
    return std::vector<std::uint8_t>(std::make_move_iterator(s.begin()), std::make_move_iterator(s.end()));
}

// ---- import ----

bool BinDb::import_from_stream(std::istream& in) {
    return import_from_stream(in, nullptr);
}

bool BinDb::import_from_stream(std::istream& in,
                                std::function<void(const std::string&, const uint8_t*, size_t)> on_replicate) {
    std::unique_lock lock(mutex_);

    char magic[4]{};
    in.read(magic, 4);
    const bool import_v2 = (std::memcmp(magic, EXPORT_MAGIC_V2, 4) == 0);
    if (!import_v2 && std::memcmp(magic, EXPORT_MAGIC, 4) != 0) return false;

    // When streaming to cluster (on_replicate set), build the store buffer
    // directly from the input stream without loading media into pending_data.
    // This avoids keeping all media bytes in memory at once.
    const bool stream_to_cluster = (on_replicate != nullptr);

    auto num_stores = bin_read_u32(in);
    std::cerr << "[import] parsing " << num_stores << " stores from buffer"
              << (stream_to_cluster ? " (streaming)" : "") << std::endl;
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        std::string store_id = bin_read_str(in);
        std::string store_name = bin_read_str(in);
        std::string store_created = bin_read_str(in);
        std::cerr << "[import] store " << si << ": id=" << store_id
                  << " name=" << store_name << std::endl;

        bool new_store = (stores_.find(store_id) == stores_.end());
        if (new_store) {
            StoreRecord s;
            s.id = store_id;
            s.name = store_name;
            s.created_at = store_created;
            stores_[store_id] = s;
        }

        // Ensure MDS3 store file exists (for non-cluster append)
        if (!stream_to_cluster) {
            ensure_store_file_locked(store_id);
        }

        // When streaming, build the MDS2 store buffer on the fly
        std::ostringstream store_buf;
        if (stream_to_cluster) {
            store_buf.write(STORE_MAGIC_V2, 4);
            // Placeholder for album count — will be patched below
            bin_write_u32(store_buf, 0);
        }
        std::uint32_t album_count = 0;

        auto num_albums = bin_read_u32(in);
        std::cerr << "[import]   store " << store_name << ": " << num_albums << " albums" << std::endl;
        for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
            std::string album_id = bin_read_str(in);
            std::string album_name = bin_read_str(in);
            std::string album_created = bin_read_str(in);

            bool album_is_public = false;
            if (import_v2) {
                std::uint8_t pub = 0;
                in.read(reinterpret_cast<char*>(&pub), 1);
                album_is_public = (pub != 0);
            }

            bool new_album = (albums_.find(album_id) == albums_.end());
            if (new_album) {
                AlbumRecord a;
                a.id = album_id;
                a.name = album_name;
                a.store_id = store_id;
                a.created_at = album_created;
                a.is_public = album_is_public;
                albums_[album_id] = a;
                stores_[store_id].album_ids.push_back(album_id);
                if (!stream_to_cluster) {
                    append_album_locked(store_id, albums_[album_id]);
                }
            }

            // Start writing album to store buffer
            if (stream_to_cluster) {
                bin_write_str(store_buf, album_id);
                bin_write_str(store_buf, album_name);
                bin_write_str(store_buf, album_created);
                { std::uint8_t pub = album_is_public ? 1 : 0;
                  store_buf.write(reinterpret_cast<const char*>(&pub), 1); }
            }

            auto num_media = bin_read_u32(in);
            std::cerr << "[import]     album " << album_name << ": " << num_media << " media" << std::endl;

            if (stream_to_cluster) {
                bin_write_u32(store_buf, num_media);
            }

            for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
                std::string media_id = bin_read_str(in);
                std::string orig_filename = bin_read_str(in);
                std::string media_kind = bin_read_str(in);
                std::string content_type = bin_read_str(in);
                std::string created_at = bin_read_str(in);
                std::uint64_t data_size = bin_read_u64(in);

                std::cerr << "[import]       media " << orig_filename
                          << " size=" << data_size << " new=" << (media_.find(media_id) == media_.end())
                          << " stream_ok=" << in.good() << std::endl;

                bool new_media = (media_.find(media_id) == media_.end());

                if (stream_to_cluster) {
                    // Write media metadata to store buffer
                    bin_write_str(store_buf, media_id);
                    bin_write_str(store_buf, orig_filename);
                    bin_write_str(store_buf, media_kind);
                    bin_write_str(store_buf, content_type);
                    bin_write_str(store_buf, created_at);
                    bin_write_u64(store_buf, data_size);

                    // Read media data from input once, write to store buffer
                    // and keep in pending_data for local access
                    std::vector<std::uint8_t> media_data;
                    if (data_size > 0) {
                        media_data.resize(data_size);
                        in.read(reinterpret_cast<char*>(media_data.data()),
                                static_cast<std::streamsize>(data_size));
                        auto got = static_cast<std::uint64_t>(in.gcount());
                        if (got < data_size) media_data.resize(got);
                        store_buf.write(reinterpret_cast<const char*>(media_data.data()),
                                        static_cast<std::streamsize>(media_data.size()));
                    }

                    // Register metadata + data locally
                    if (new_media) {
                        MediaRecord m;
                        m.id = media_id;
                        m.original_filename = orig_filename;
                        m.media_kind = media_kind;
                        m.content_type = content_type;
                        m.created_at = created_at;
                        m.size_bytes = data_size;
                        m.album_id = album_id;
                        m.store_id = store_id;
                        m.pending_data = std::move(media_data);
                        media_[media_id] = std::move(m);
                        auto& mids = albums_[album_id].media_ids;
                        if (std::find(mids.begin(), mids.end(), media_id) == mids.end()) {
                            mids.push_back(media_id);
                        }
                    }
                } else {
                    // Non-cluster import: append directly to MDS3 store file
                    if (new_media) {
                        MediaRecord m;
                        m.id = media_id;
                        m.original_filename = orig_filename;
                        m.media_kind = media_kind;
                        m.content_type = content_type;
                        m.created_at = created_at;
                        m.size_bytes = data_size;
                        m.album_id = album_id;
                        m.store_id = store_id;
                        if (data_size > 0) {
                            m.pending_data.resize(data_size);
                            in.read(reinterpret_cast<char*>(m.pending_data.data()),
                                    static_cast<std::streamsize>(data_size));
                        }
                        media_[media_id] = std::move(m);
                        auto& mids = albums_[album_id].media_ids;
                        if (std::find(mids.begin(), mids.end(), media_id) == mids.end()) {
                            mids.push_back(media_id);
                        }
                        append_media_locked(store_id, media_[media_id]);
                    } else {
                        if (data_size > 0) {
                            in.seekg(static_cast<std::streamoff>(data_size), std::ios::cur);
                        }
                    }
                }
            }

            album_count++;
        }

        // Non-cluster: final mmap after all appends for this store
        if (!stream_to_cluster && !memory_only_) {
            mmap_store(store_id);
        }

        // Replicate this store directly to cluster after parsing
        if (stream_to_cluster) {
            // Patch album count at offset 4 (after STORE_MAGIC_V2)
            std::string buf_str = store_buf.str();
            std::memcpy(buf_str.data() + 4, &album_count, 4);
            std::cerr << "[import] replicate store:" << store_id
                      << " buf=" << buf_str.size() << " bytes (streamed)" << std::endl;
            on_replicate("store:" + store_id,
                         reinterpret_cast<const uint8_t*>(buf_str.data()),
                         buf_str.size());
        }

        save_index_locked();
    }

    std::cerr << "[import] done: " << stores_.size() << " stores, "
              << albums_.size() << " albums, "
              << media_.size() << " media" << std::endl;

    // Replicate final index to cluster
    if (on_replicate) {
        auto buf = save_index_to_buffer_locked();
        if (!buf.empty()) {
            on_replicate("index", buf.data(), buf.size());
        }
    }

    return true;
}

bool BinDb::import_db(const std::string& path) {
    std::ifstream in(path, std::ios::binary);
    if (!in.is_open()) return false;
    return import_from_stream(in);
}

bool BinDb::import_db_from_buffer(const std::vector<std::uint8_t>& data) {
    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);
    return import_from_stream(in);
}

bool BinDb::import_db_from_buffer(const std::vector<std::uint8_t>& data,
                                   std::function<void(const std::string&, const uint8_t*, size_t)> on_replicate) {
    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);
    return import_from_stream(in, std::move(on_replicate));
}

// ---- reload / store_ids ----

void BinDb::reload() {
    if (memory_only_) return;  // memory-only mode has no files to reload from
    std::unique_lock lock(mutex_);
    stores_.clear();
    albums_.clear();
    media_.clear();
    mmaps_.clear();
    load_index();
    for (auto& [sid, store] : stores_) {
        load_store(sid, store);
    }
    mmap_all_stores();
}

std::vector<std::string> BinDb::store_ids() const {
    std::shared_lock lock(mutex_);
    std::vector<std::string> ids;
    ids.reserve(stores_.size());
    for (const auto& [k, _] : stores_) ids.push_back(k);
    return ids;
}

// ---- load ----

void BinDb::load() {
    if (memory_only_) return;
    load_index();
    for (auto& [sid, store] : stores_) {
        load_store(sid, store);
    }
    mmap_all_stores();
}

void BinDb::load_index() {
    std::ifstream in(index_file(), std::ios::binary);
    if (!in.is_open()) return;

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v4 = (std::memcmp(magic, INDEX_MAGIC, 4) == 0);
    const bool is_v3 = is_v4 || (std::memcmp(magic, INDEX_MAGIC_V3, 4) == 0);
    if (!is_v3 && std::memcmp(magic, INDEX_MAGIC_V2, 4) != 0) return;

    auto num_stores = bin_read_u32(in);
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        StoreRecord s;
        s.id = bin_read_str(in);
        s.name = bin_read_str(in);
        s.created_at = bin_read_str(in);
        if (is_v3) {
            auto num_domains = bin_read_u32(in);
            for (std::uint32_t di = 0; di < num_domains && in.good(); ++di) {
                AuthDomain d;
                d.id = bin_read_str(in);
                d.label = bin_read_str(in);
                d.url = bin_read_str(in);
                d.client_name = bin_read_str(in);
                d.client_secret = bin_read_str(in);
                s.auth_domains.push_back(std::move(d));
            }
        }
        if (is_v4) {
            auto num_acls = bin_read_u32(in);
            for (std::uint32_t ai = 0; ai < num_acls && in.good(); ++ai) {
                StoreACL acl;
                acl.id = bin_read_str(in);
                acl.group_id = bin_read_str(in);
                acl.group_name = bin_read_str(in);
                std::uint8_t flags = 0;
                in.read(reinterpret_cast<char*>(&flags), 1);
                acl.can_read  = (flags & 0x01) != 0;
                acl.can_write = (flags & 0x02) != 0;
                s.acls.push_back(std::move(acl));
            }
        }
        stores_[s.id] = std::move(s);
    }
}

void BinDb::load_store(const std::string& store_id, StoreRecord& store) {
    std::ifstream in(store_file(store_id), std::ios::binary);
    if (!in.is_open()) return;

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v3 = (std::memcmp(magic, STORE_MAGIC_V3, 4) == 0);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v3 && !is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    if (is_v3) {
        // MDS3: flat record stream with 4 bytes reserved padding after magic
        in.seekg(8);
        while (in.good()) {
            auto rec_off = static_cast<std::uint64_t>(in.tellg());
            std::uint8_t rec_type = 0;
            in.read(reinterpret_cast<char*>(&rec_type), 1);
            if (!in.good()) break;
            std::uint64_t body_size = 0;
            in.read(reinterpret_cast<char*>(&body_size), 8);
            if (!in.good()) break;

            // Next record starts after header (9 bytes) + body
            auto next_rec = static_cast<std::streamoff>(rec_off + 9 + body_size);

            if (rec_type == REC_ALBUM) {
                AlbumRecord a;
                a.id = bin_read_str(in);
                a.name = bin_read_str(in);
                a.created_at = bin_read_str(in);
                { std::uint8_t pub = 0;
                  in.read(reinterpret_cast<char*>(&pub), 1);
                  a.is_public = (pub != 0); }
                a.store_id = store_id;
                a.record_offset = rec_off;

                // Later album records with same ID overwrite earlier ones (for edits)
                auto existing = albums_.find(a.id);
                if (existing != albums_.end()) {
                    a.media_ids = std::move(existing->second.media_ids);
                    albums_[a.id] = std::move(a);
                } else {
                    albums_[a.id] = std::move(a);
                    store.album_ids.push_back(albums_[a.id].id);
                }
            } else if (rec_type == REC_MEDIA) {
                MediaRecord m;
                m.id = bin_read_str(in);
                m.album_id = bin_read_str(in);
                m.original_filename = bin_read_str(in);
                m.media_kind = bin_read_str(in);
                m.content_type = bin_read_str(in);
                m.created_at = bin_read_str(in);
                m.size_bytes = bin_read_u64(in);
                m.store_id = store_id;
                m.record_offset = rec_off;
                m.data_offset = static_cast<std::uint64_t>(in.tellg());
                // Skip raw data — don't read it into memory

                auto ait = albums_.find(m.album_id);
                if (ait != albums_.end()) {
                    auto& mids = ait->second.media_ids;
                    if (std::find(mids.begin(), mids.end(), m.id) == mids.end()) {
                        mids.push_back(m.id);
                    }
                }
                media_[m.id] = std::move(m);
            }
            // Jump to next record (skips REC_EMPTY, unknown types,
            // and any remaining body bytes not consumed above)
            in.seekg(next_rec);
        }
        return;
    }

    // MDS1/MDS2: hierarchical format (legacy)
    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;
            m.data_offset = static_cast<std::uint64_t>(in.tellg());
            in.seekg(static_cast<std::streamoff>(m.size_bytes), std::ios::cur);

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        store.album_ids.push_back(a.id);
        albums_[a.id] = std::move(a);
    }
}

// ---- save ----

void BinDb::save_index_locked() const {
    if (memory_only_) return;
    const std::string tmp = index_file().string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;
        out.write(INDEX_MAGIC, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(stores_.size()));
        for (const auto& [sid, s] : stores_) {
            bin_write_str(out, s.id);
            bin_write_str(out, s.name);
            bin_write_str(out, s.created_at);
            bin_write_u32(out, static_cast<std::uint32_t>(s.auth_domains.size()));
            for (const auto& d : s.auth_domains) {
                bin_write_str(out, d.id);
                bin_write_str(out, d.label);
                bin_write_str(out, d.url);
                bin_write_str(out, d.client_name);
                bin_write_str(out, d.client_secret);
            }
            bin_write_u32(out, static_cast<std::uint32_t>(s.acls.size()));
            for (const auto& acl : s.acls) {
                bin_write_str(out, acl.id);
                bin_write_str(out, acl.group_id);
                bin_write_str(out, acl.group_name);
                std::uint8_t flags = 0;
                if (acl.can_read)  flags |= 0x01;
                if (acl.can_write) flags |= 0x02;
                out.write(reinterpret_cast<const char*>(&flags), 1);
            }
        }
    }
    fs::rename(tmp, index_file());
}

void BinDb::save_store_locked(const std::string& store_id) {
    if (memory_only_) return;  // in memory-only mode, data stays in pending_data
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    const auto& store = sit->second;

    const fs::path sf = store_file(store_id);
    std::ifstream old_in(sf, std::ios::binary);

    const std::string tmp = sf.string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;
        out.write(STORE_MAGIC_V2, 4);
        bin_write_u32(out, static_cast<std::uint32_t>(store.album_ids.size()));

        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) continue;
            const auto& a = ait->second;

            bin_write_str(out, a.id);
            bin_write_str(out, a.name);
            bin_write_str(out, a.created_at);
            { std::uint8_t pub = a.is_public ? 1 : 0; out.write(reinterpret_cast<const char*>(&pub), 1); }
            bin_write_u32(out, static_cast<std::uint32_t>(a.media_ids.size()));

            for (const auto& mid : a.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                auto& m = mit->second;

                bin_write_str(out, m.id);
                bin_write_str(out, m.original_filename);
                bin_write_str(out, m.media_kind);
                bin_write_str(out, m.content_type);
                bin_write_str(out, m.created_at);
                bin_write_u64(out, m.size_bytes);

                const auto new_offset = static_cast<std::uint64_t>(out.tellp());

                if (!m.pending_data.empty()) {
                    out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                    m.pending_data.clear();
                    m.pending_data.shrink_to_fit();
                } else if (old_in.is_open() && m.data_offset > 0 && m.size_bytes > 0) {
                    old_in.seekg(static_cast<std::streamoff>(m.data_offset));
                    constexpr std::size_t BUF = 1024 * 1024;
                    std::vector<char> buf(BUF);
                    std::uint64_t remaining = m.size_bytes;
                    while (remaining > 0 && old_in.good()) {
                        auto chunk = static_cast<std::streamsize>(std::min<std::uint64_t>(remaining, BUF));
                        old_in.read(buf.data(), chunk);
                        auto got = old_in.gcount();
                        if (got > 0) out.write(buf.data(), got);
                        remaining -= static_cast<std::uint64_t>(got);
                        if (got < chunk) break;
                    }
                }

                m.data_offset = new_offset;
            }
        }
    }
    old_in.close();
    fs::rename(tmp, sf);
    mmap_store(store_id);
}

// ---- MDS3: append-only store operations ----

void BinDb::ensure_store_file_locked(const std::string& store_id) {
    if (memory_only_) return;
    const fs::path sf = store_file(store_id);
    if (fs::exists(sf)) return;
    std::ofstream out(sf, std::ios::binary | std::ios::trunc);
    if (!out.is_open()) return;
    out.write(STORE_MAGIC_V3, 4);
    std::uint32_t reserved = 0;
    out.write(reinterpret_cast<const char*>(&reserved), 4);
}

void BinDb::append_album_locked(const std::string& store_id, const AlbumRecord& album) {
    if (memory_only_) return;
    ensure_store_file_locked(store_id);
    const fs::path sf = store_file(store_id);
    std::ofstream out(sf, std::ios::binary | std::ios::app);
    if (!out.is_open()) return;

    // Build album body
    std::ostringstream body(std::ios::binary);
    bin_write_str(body, album.id);
    bin_write_str(body, album.name);
    bin_write_str(body, album.created_at);
    { std::uint8_t pub = album.is_public ? 1 : 0;
      body.write(reinterpret_cast<const char*>(&pub), 1); }
    std::string body_str = body.str();

    // Record offset is the current end of file
    auto rec_off = static_cast<std::uint64_t>(out.tellp());

    // Write record header: type(u8) + body_size(u64)
    out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
    auto bsz = static_cast<std::uint64_t>(body_str.size());
    out.write(reinterpret_cast<const char*>(&bsz), 8);
    out.write(body_str.data(), static_cast<std::streamsize>(body_str.size()));

    // Update in-memory record offset
    auto ait = albums_.find(album.id);
    if (ait != albums_.end()) ait->second.record_offset = rec_off;

    mmap_store(store_id);
}

void BinDb::append_media_locked(const std::string& store_id, MediaRecord& media) {
    if (memory_only_) return;
    ensure_store_file_locked(store_id);
    const fs::path sf = store_file(store_id);
    std::ofstream out(sf, std::ios::binary | std::ios::app);
    if (!out.is_open()) return;

    // Build media body (metadata portion)
    std::ostringstream meta(std::ios::binary);
    bin_write_str(meta, media.id);
    bin_write_str(meta, media.album_id);
    bin_write_str(meta, media.original_filename);
    bin_write_str(meta, media.media_kind);
    bin_write_str(meta, media.content_type);
    bin_write_str(meta, media.created_at);
    bin_write_u64(meta, media.size_bytes);
    std::string meta_str = meta.str();

    auto body_size = static_cast<std::uint64_t>(meta_str.size()) + media.size_bytes;

    // Record offset
    auto rec_off = static_cast<std::uint64_t>(out.tellp());

    // Write record header: type(u8) + body_size(u64)
    out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
    out.write(reinterpret_cast<const char*>(&body_size), 8);

    // Write metadata
    out.write(meta_str.data(), static_cast<std::streamsize>(meta_str.size()));

    // data_offset = current file position (start of raw data)
    auto data_off = static_cast<std::uint64_t>(out.tellp());

    // Write raw data
    if (!media.pending_data.empty()) {
        out.write(reinterpret_cast<const char*>(media.pending_data.data()),
                  static_cast<std::streamsize>(media.pending_data.size()));
        media.pending_data.clear();
        media.pending_data.shrink_to_fit();
    }

    media.record_offset = rec_off;
    media.data_offset = data_off;

    mmap_store(store_id);
}

void BinDb::zero_record_locked(const std::string& store_id, std::uint64_t record_offset) {
    if (memory_only_) return;
    if (record_offset == 0) return;
    const fs::path sf = store_file(store_id);
    int fd = ::open(sf.c_str(), O_RDWR);
    if (fd < 0) return;

    // Mark as deleted: single byte write (type = REC_EMPTY)
    // body_size stays intact so load/vacuum can skip over the record
    std::uint8_t empty_type = REC_EMPTY;
    ::pwrite(fd, &empty_type, 1, static_cast<off_t>(record_offset));

    ::close(fd);
    mmap_store(store_id);
}

void BinDb::vacuum_store_locked(const std::string& store_id) {
    if (memory_only_) return;
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    const auto& store = sit->second;
    const fs::path sf = store_file(store_id);
    if (!fs::exists(sf)) return;

    const std::string tmp = sf.string() + ".tmp";
    {
        std::ofstream out(tmp, std::ios::binary | std::ios::trunc);
        if (!out.is_open()) return;

        // Write MDS3 header
        out.write(STORE_MAGIC_V3, 4);
        std::uint32_t reserved = 0;
        out.write(reinterpret_cast<const char*>(&reserved), 4);

        std::ifstream old_in(sf, std::ios::binary);

        // Write all valid albums
        for (const auto& aid : store.album_ids) {
            auto ait = albums_.find(aid);
            if (ait == albums_.end()) continue;
            auto& a = ait->second;

            std::ostringstream body(std::ios::binary);
            bin_write_str(body, a.id);
            bin_write_str(body, a.name);
            bin_write_str(body, a.created_at);
            { std::uint8_t pub = a.is_public ? 1 : 0;
              body.write(reinterpret_cast<const char*>(&pub), 1); }
            std::string body_str = body.str();

            a.record_offset = static_cast<std::uint64_t>(out.tellp());
            out.write(reinterpret_cast<const char*>(&REC_ALBUM), 1);
            auto bsz = static_cast<std::uint64_t>(body_str.size());
            out.write(reinterpret_cast<const char*>(&bsz), 8);
            out.write(body_str.data(), static_cast<std::streamsize>(body_str.size()));

            // Write all valid media in this album
            for (const auto& mid : a.media_ids) {
                auto mit = media_.find(mid);
                if (mit == media_.end()) continue;
                auto& m = mit->second;

                std::ostringstream mbody(std::ios::binary);
                bin_write_str(mbody, m.id);
                bin_write_str(mbody, m.album_id);
                bin_write_str(mbody, m.original_filename);
                bin_write_str(mbody, m.media_kind);
                bin_write_str(mbody, m.content_type);
                bin_write_str(mbody, m.created_at);
                bin_write_u64(mbody, m.size_bytes);
                std::string mbody_str = mbody.str();

                auto total_body = static_cast<std::uint64_t>(mbody_str.size()) + m.size_bytes;

                m.record_offset = static_cast<std::uint64_t>(out.tellp());
                out.write(reinterpret_cast<const char*>(&REC_MEDIA), 1);
                out.write(reinterpret_cast<const char*>(&total_body), 8);
                out.write(mbody_str.data(), static_cast<std::streamsize>(mbody_str.size()));

                auto old_data_offset = m.data_offset; // save before vacuum overwrites

                m.data_offset = static_cast<std::uint64_t>(out.tellp());

                // Copy raw data from old file or pending_data
                if (!m.pending_data.empty()) {
                    out.write(reinterpret_cast<const char*>(m.pending_data.data()),
                              static_cast<std::streamsize>(m.pending_data.size()));
                    m.pending_data.clear();
                    m.pending_data.shrink_to_fit();
                } else if (old_in.is_open() && m.size_bytes > 0) {
                    old_in.seekg(static_cast<std::streamoff>(old_data_offset));
                    constexpr std::size_t BUF = 1024 * 1024;
                    std::vector<char> buf(BUF);
                    std::uint64_t rem = m.size_bytes;
                    while (rem > 0 && old_in.good()) {
                        auto chunk = static_cast<std::streamsize>(std::min<std::uint64_t>(rem, BUF));
                        old_in.read(buf.data(), chunk);
                        auto got = old_in.gcount();
                        if (got > 0) out.write(buf.data(), got);
                        rem -= static_cast<std::uint64_t>(got);
                        if (got < chunk) break;
                    }
                }
            }
        }
    }
    fs::rename(tmp, sf);
    mmap_store(store_id);
    std::cerr << "[vacuum] store " << store_id << " compacted" << std::endl;
}

void BinDb::vacuum_store(const std::string& store_id) {
    std::unique_lock lock(mutex_);
    vacuum_store_locked(store_id);
}

void BinDb::vacuum_all_stores() {
    std::unique_lock lock(mutex_);
    for (const auto& [sid, _] : stores_) {
        vacuum_store_locked(sid);
    }
}

// ---- mmap management ----

void BinDb::mmap_store(const std::string& store_id) {
    auto path = store_file(store_id);
    if (fs::exists(path) && fs::file_size(path) > 0) {
        mmaps_[store_id] = MmapRegion::open(path);
    }
}

void BinDb::mmap_all_stores() {
    for (const auto& [sid, _] : stores_) {
        mmap_store(sid);
    }
}

// ---- buffer-based save/load (for cluster-only storage) ----

std::vector<std::uint8_t> BinDb::save_index_to_buffer() const {
    std::shared_lock lock(mutex_);
    return save_index_to_buffer_locked();
}

std::vector<std::uint8_t> BinDb::save_index_to_buffer_locked() const {
    std::ostringstream oss(std::ios::binary);
    oss.write(INDEX_MAGIC, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(stores_.size()));
    for (const auto& [sid, s] : stores_) {
        bin_write_str(oss, s.id);
        bin_write_str(oss, s.name);
        bin_write_str(oss, s.created_at);
        bin_write_u32(oss, static_cast<std::uint32_t>(s.auth_domains.size()));
        for (const auto& d : s.auth_domains) {
            bin_write_str(oss, d.id);
            bin_write_str(oss, d.label);
            bin_write_str(oss, d.url);
            bin_write_str(oss, d.client_name);
            bin_write_str(oss, d.client_secret);
        }
        bin_write_u32(oss, static_cast<std::uint32_t>(s.acls.size()));
        for (const auto& acl : s.acls) {
            bin_write_str(oss, acl.id);
            bin_write_str(oss, acl.group_id);
            bin_write_str(oss, acl.group_name);
            std::uint8_t flags = 0;
            if (acl.can_read)  flags |= 0x01;
            if (acl.can_write) flags |= 0x02;
            oss.write(reinterpret_cast<const char*>(&flags), 1);
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_index_from_buffer(const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v4 = (std::memcmp(magic, INDEX_MAGIC, 4) == 0);
    const bool is_v3 = is_v4 || (std::memcmp(magic, INDEX_MAGIC_V3, 4) == 0);
    if (!is_v3 && std::memcmp(magic, INDEX_MAGIC_V2, 4) != 0) return;

    // Preserve existing store IDs to detect removed stores
    std::set<std::string> old_ids;
    for (const auto& [k, _] : stores_) old_ids.insert(k);

    auto num_stores = bin_read_u32(in);
    for (std::uint32_t si = 0; si < num_stores && in.good(); ++si) {
        StoreRecord s;
        s.id = bin_read_str(in);
        s.name = bin_read_str(in);
        s.created_at = bin_read_str(in);
        if (is_v3) {
            auto num_domains = bin_read_u32(in);
            for (std::uint32_t di = 0; di < num_domains && in.good(); ++di) {
                AuthDomain d;
                d.id = bin_read_str(in);
                d.label = bin_read_str(in);
                d.url = bin_read_str(in);
                d.client_name = bin_read_str(in);
                d.client_secret = bin_read_str(in);
                s.auth_domains.push_back(std::move(d));
            }
        }
        if (is_v4) {
            auto num_acls = bin_read_u32(in);
            for (std::uint32_t ai = 0; ai < num_acls && in.good(); ++ai) {
                StoreACL acl;
                acl.id = bin_read_str(in);
                acl.group_id = bin_read_str(in);
                acl.group_name = bin_read_str(in);
                std::uint8_t flags = 0;
                in.read(reinterpret_cast<char*>(&flags), 1);
                acl.can_read  = (flags & 0x01) != 0;
                acl.can_write = (flags & 0x02) != 0;
                s.acls.push_back(std::move(acl));
            }
        }
        // Keep existing album_ids if store already present
        auto existing = stores_.find(s.id);
        if (existing != stores_.end()) {
            s.album_ids = existing->second.album_ids;
        }
        stores_[s.id] = std::move(s);
        old_ids.erase(s.id);
    }

    // Do not remove locally-present stores that are absent from the buffer.
    // Deletes propagate explicitly via delete_store() + cluster_.remove();
    // removing here would wipe out recently-created local stores when the
    // synced index is stale (same merge-only approach as authdb).
}

std::vector<std::uint8_t> BinDb::save_store_to_buffer(const std::string& store_id) {
    std::shared_lock lock(mutex_);
    return save_store_to_buffer_locked(store_id);
}

std::vector<std::uint8_t> BinDb::save_store_to_buffer_locked(const std::string& store_id) const {
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return {};
    const auto& store = sit->second;

    std::ostringstream oss(std::ios::binary);
    oss.write(STORE_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(store.album_ids.size()));

    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait == albums_.end()) continue;
        const auto& a = ait->second;

        bin_write_str(oss, a.id);
        bin_write_str(oss, a.name);
        bin_write_str(oss, a.created_at);
        { std::uint8_t pub = a.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
        bin_write_u32(oss, static_cast<std::uint32_t>(a.media_ids.size()));

        for (const auto& mid : a.media_ids) {
            auto mit = media_.find(mid);
            if (mit == media_.end()) continue;
            const auto& m = mit->second;

            bin_write_str(oss, m.id);
            bin_write_str(oss, m.original_filename);
            bin_write_str(oss, m.media_kind);
            bin_write_str(oss, m.content_type);
            bin_write_str(oss, m.created_at);
            bin_write_u64(oss, m.size_bytes);

            // In memory-only mode, data is always in pending_data
            if (!m.pending_data.empty()) {
                oss.write(reinterpret_cast<const char*>(m.pending_data.data()),
                          static_cast<std::streamsize>(m.pending_data.size()));
            }
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_store_from_buffer(const std::string& store_id, const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    auto& store = sit->second;

    // Clear old albums/media for this store
    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) {
            for (const auto& mid : ait->second.media_ids) media_.erase(mid);
            albums_.erase(ait);
        }
    }
    store.album_ids.clear();

    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;

            // Read media binary data into pending_data (memory-only)
            if (m.size_bytes > 0) {
                m.pending_data.resize(m.size_bytes);
                in.read(reinterpret_cast<char*>(m.pending_data.data()),
                        static_cast<std::streamsize>(m.size_bytes));
            }
            m.data_offset = 0;

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        store.album_ids.push_back(a.id);
        albums_[a.id] = std::move(a);
    }
}

// ==================== ClusterMediaBackend ====================

// ---- metadata-only store buffer (no raw media data) ----

std::vector<std::uint8_t> BinDb::save_store_metadata_to_buffer(const std::string& store_id) const {
    std::shared_lock lock(mutex_);
    return save_store_metadata_to_buffer_locked(store_id);
}

std::vector<std::uint8_t> BinDb::save_store_metadata_to_buffer_locked(const std::string& store_id) const {
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return {};
    const auto& store = sit->second;

    std::ostringstream oss(std::ios::binary);
    oss.write(STORE_MAGIC_V2, 4);
    bin_write_u32(oss, static_cast<std::uint32_t>(store.album_ids.size()));

    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait == albums_.end()) continue;
        const auto& a = ait->second;

        bin_write_str(oss, a.id);
        bin_write_str(oss, a.name);
        bin_write_str(oss, a.created_at);
        { std::uint8_t pub = a.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }
        bin_write_u32(oss, static_cast<std::uint32_t>(a.media_ids.size()));

        for (const auto& mid : a.media_ids) {
            auto mit = media_.find(mid);
            if (mit == media_.end()) continue;
            const auto& m = mit->second;

            bin_write_str(oss, m.id);
            bin_write_str(oss, m.original_filename);
            bin_write_str(oss, m.media_kind);
            bin_write_str(oss, m.content_type);
            bin_write_str(oss, m.created_at);
            bin_write_u64(oss, m.size_bytes);
            // NO raw data written — metadata only
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

void BinDb::load_store_metadata_from_buffer(const std::string& store_id, const std::vector<std::uint8_t>& data) {
    std::unique_lock lock(mutex_);
    auto sit = stores_.find(store_id);
    if (sit == stores_.end()) return;
    auto& store = sit->second;

    // Clear old albums/media for this store
    for (const auto& aid : store.album_ids) {
        auto ait = albums_.find(aid);
        if (ait != albums_.end()) {
            for (const auto& mid : ait->second.media_ids) media_.erase(mid);
            albums_.erase(ait);
        }
    }
    store.album_ids.clear();

    std::string str(data.begin(), data.end());
    std::istringstream in(str, std::ios::binary);

    char magic[4]{};
    in.read(magic, 4);
    const bool is_v2 = (std::memcmp(magic, STORE_MAGIC_V2, 4) == 0);
    if (!is_v2 && std::memcmp(magic, STORE_MAGIC, 4) != 0) return;

    auto num_albums = bin_read_u32(in);
    for (std::uint32_t ai = 0; ai < num_albums && in.good(); ++ai) {
        AlbumRecord a;
        a.id = bin_read_str(in);
        a.name = bin_read_str(in);
        a.store_id = store_id;
        a.created_at = bin_read_str(in);
        if (is_v2) {
            std::uint8_t pub = 0;
            in.read(reinterpret_cast<char*>(&pub), 1);
            a.is_public = (pub != 0);
        }

        auto num_media = bin_read_u32(in);
        for (std::uint32_t mi = 0; mi < num_media && in.good(); ++mi) {
            MediaRecord m;
            m.id = bin_read_str(in);
            m.original_filename = bin_read_str(in);
            m.media_kind = bin_read_str(in);
            m.content_type = bin_read_str(in);
            m.created_at = bin_read_str(in);
            m.size_bytes = bin_read_u64(in);
            m.album_id = a.id;
            m.store_id = store_id;
            // NO raw data — media data lives in cluster only
            m.data_offset = 0;

            a.media_ids.push_back(m.id);
            media_[m.id] = std::move(m);
        }

        store.album_ids.push_back(a.id);
        albums_[a.id] = std::move(a);
    }
}

std::optional<MediaRecord> BinDb::add_media_metadata(const std::string& album_id,
                                                      const std::string& filename,
                                                      const std::string& content_type,
                                                      std::uint64_t size_bytes) {
    std::unique_lock lock(mutex_);
    auto ait = albums_.find(album_id);
    if (ait == albums_.end()) return std::nullopt;

    const fs::path p(filename);
    MediaRecord m;
    m.id = make_uuid();
    m.original_filename = filename;
    m.media_kind = detect_kind(p.extension().string());
    m.content_type = content_type;
    m.album_id = album_id;
    m.store_id = ait->second.store_id;
    m.created_at = now_iso8601();
    m.size_bytes = size_bytes;
    m.data_offset = 0;
    // no pending_data — raw data goes directly to cluster

    const std::string mid = m.id;
    media_[mid] = std::move(m);
    ait->second.media_ids.push_back(mid);

    auto mit = media_.find(mid);
    if (mit == media_.end()) return std::nullopt;
    return mit->second;
}

ClusterMediaBackend::ClusterMediaBackend(Cluster& cluster)
    : local_(), cluster_(cluster) {
    // Load initial metadata from cluster (no local raw data)
    sync_from_cluster();
    sync_running_ = true;
    sync_thread_ = std::thread(&ClusterMediaBackend::sync_loop, this);
}

ClusterMediaBackend::~ClusterMediaBackend() {
    sync_running_ = false;
    sync_cv_.notify_all();
    if (sync_thread_.joinable()) sync_thread_.join();
}

// ---- stores ----
StoreRecord ClusterMediaBackend::create_store(const std::string& name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto s = local_.create_store(name);
    replicate_store(s.id);
    replicate_index();
    return s;
}

std::optional<StoreRecord> ClusterMediaBackend::get_store(const std::string& id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_store(id);
}

std::vector<StoreRecord> ClusterMediaBackend::list_stores() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_stores();
}

bool ClusterMediaBackend::delete_store(const std::string& id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.delete_store(id)) return false;
    cluster_.remove("store:" + id);
    replicate_index();
    return true;
}

bool ClusterMediaBackend::rename_store(const std::string& id, const std::string& new_name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.rename_store(id, new_name)) return false;
    replicate_index();
    return true;
}

// ---- store auth domains ----
std::string ClusterMediaBackend::add_store_domain(const std::string& store_id, AuthDomain domain) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto did = local_.add_store_domain(store_id, std::move(domain));
    if (!did.empty()) replicate_index();
    return did;
}

bool ClusterMediaBackend::remove_store_domain(const std::string& store_id, const std::string& domain_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.remove_store_domain(store_id, domain_id)) return false;
    replicate_index();
    return true;
}

std::vector<AuthDomain> ClusterMediaBackend::list_store_domains(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_store_domains(store_id);
}

std::optional<AuthDomain> ClusterMediaBackend::find_domain_by_label(const std::string& label) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.find_domain_by_label(label);
}

// ---- store ACLs ----
std::string ClusterMediaBackend::add_store_acl(const std::string& store_id, StoreACL acl) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto aid = local_.add_store_acl(store_id, std::move(acl));
    if (!aid.empty()) replicate_index();
    return aid;
}

bool ClusterMediaBackend::remove_store_acl(const std::string& store_id, const std::string& acl_id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.remove_store_acl(store_id, acl_id)) return false;
    replicate_index();
    return true;
}

std::vector<StoreACL> ClusterMediaBackend::list_store_acls(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_store_acls(store_id);
}

std::vector<std::string> ClusterMediaBackend::all_acl_group_ids() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.all_acl_group_ids();
}

// ---- albums ----
std::optional<AlbumRecord> ClusterMediaBackend::create_album(const std::string& store_id, const std::string& name) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto a = local_.create_album(store_id, name);
    if (a) replicate_store(store_id);
    return a;
}

std::optional<AlbumRecord> ClusterMediaBackend::get_album(const std::string& id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_album(id);
}

std::vector<AlbumRecord> ClusterMediaBackend::list_albums(const std::string& store_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_albums(store_id);
}

bool ClusterMediaBackend::delete_album(const std::string& id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto album = local_.get_album(id);
    if (!album) return false;
    std::string store_id = album->store_id;
    if (!local_.delete_album(id)) return false;
    replicate_store(store_id);
    replicate_index();
    return true;
}

bool ClusterMediaBackend::set_album_public(const std::string& album_id, bool is_public) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto album = local_.get_album(album_id);
    if (!album) return false;
    if (!local_.set_album_public(album_id, is_public)) return false;
    replicate_store(album->store_id);
    return true;
}

bool ClusterMediaBackend::is_album_public(const std::string& album_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.is_album_public(album_id);
}

bool ClusterMediaBackend::is_media_public(const std::string& media_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.is_media_public(media_id);
}

std::vector<AlbumRecord> ClusterMediaBackend::list_public_albums() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_public_albums();
}

// ---- media ----
std::optional<MediaRecord> ClusterMediaBackend::add_media(const std::string& album_id,
                                                            const std::string& filename,
                                                            const std::string& content_type,
                                                            std::vector<std::uint8_t> data) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    // Store metadata only (no raw data locally)
    auto m = local_.add_media_metadata(album_id, filename, content_type, data.size());
    if (m) {
        // Replicate raw data directly to cluster under "media:{id}"
        replicate_media(m->id, data.data(), data.size());
        // Replicate lightweight metadata-only store
        replicate_store(m->store_id);
    }
    return m;
}

std::optional<MediaRecord> ClusterMediaBackend::get_media(const std::string& id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_media(id);
}

std::vector<MediaRecord> ClusterMediaBackend::list_media(const std::string& album_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.list_media(album_id);
}

bool ClusterMediaBackend::delete_media(const std::string& id) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    auto media = local_.get_media(id);
    if (!media) return false;
    std::string store_id = media->store_id;
    if (!local_.delete_media(id)) return false;
    // Remove raw data from cluster
    cluster_.remove("media:" + id);
    replicate_store(store_id);
    replicate_index();
    return true;
}

// ---- raw data access (fetch from cluster, no local data) ----
std::vector<std::uint8_t> ClusterMediaBackend::read_media_data(const std::string& media_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    std::vector<uint8_t> out;
    if (fetch_media_from_cluster(media_id, out)) return out;
    return {};
}

std::vector<std::uint8_t> ClusterMediaBackend::read_media_data_range(const std::string& media_id,
                                                                       std::uint64_t offset,
                                                                       std::uint64_t length) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    // Fetch full data from cluster and slice the range
    std::vector<uint8_t> full;
    if (!fetch_media_from_cluster(media_id, full)) return {};
    if (offset >= full.size()) return {};
    const std::uint64_t avail = full.size() - offset;
    const std::uint64_t to_read = std::min(length, avail);
    return std::vector<std::uint8_t>(
        full.begin() + static_cast<std::ptrdiff_t>(offset),
        full.begin() + static_cast<std::ptrdiff_t>(offset + to_read));
}

std::uint64_t ClusterMediaBackend::get_media_size(const std::string& media_id) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return local_.get_media_size(media_id);
}

// ---- export / import ----
bool ClusterMediaBackend::export_db(const std::string& path) const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    // Export requires raw data — fetch each media from cluster
    auto buf = export_db_to_buffer_impl();
    if (buf.empty()) return false;
    std::ofstream out(path, std::ios::binary | std::ios::trunc);
    if (!out.is_open()) return false;
    out.write(reinterpret_cast<const char*>(buf.data()),
              static_cast<std::streamsize>(buf.size()));
    return out.good();
}

std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer() const {
    std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    return export_db_to_buffer_impl();
}

std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer_impl() const {
    // Build export buffer with raw data fetched from cluster
    static constexpr char EXPORT_MAGIC_V2[4] = {'M', 'D', 'E', '2'};
    std::ostringstream oss(std::ios::binary);
    oss.write(EXPORT_MAGIC_V2, 4);

    auto stores = local_.list_stores();
    bin_write_u32(oss, static_cast<std::uint32_t>(stores.size()));

    for (const auto& store : stores) {
        bin_write_str(oss, store.id);
        bin_write_str(oss, store.name);
        bin_write_str(oss, store.created_at);

        auto albums = local_.list_albums(store.id);
        bin_write_u32(oss, static_cast<std::uint32_t>(albums.size()));

        for (const auto& album : albums) {
            bin_write_str(oss, album.id);
            bin_write_str(oss, album.name);
            bin_write_str(oss, album.created_at);
            { std::uint8_t pub = album.is_public ? 1 : 0; oss.write(reinterpret_cast<const char*>(&pub), 1); }

            auto media_list = local_.list_media(album.id);
            bin_write_u32(oss, static_cast<std::uint32_t>(media_list.size()));

            for (const auto& m : media_list) {
                bin_write_str(oss, m.id);
                bin_write_str(oss, m.original_filename);
                bin_write_str(oss, m.media_kind);
                bin_write_str(oss, m.content_type);
                bin_write_str(oss, m.created_at);
                bin_write_u64(oss, m.size_bytes);

                // Fetch raw data from cluster
                std::vector<uint8_t> raw;
                if (m.size_bytes > 0 && fetch_media_from_cluster(m.id, raw)) {
                    oss.write(reinterpret_cast<const char*>(raw.data()),
                              static_cast<std::streamsize>(raw.size()));
                } else if (m.size_bytes > 0) {
                    // Pad with zeros if fetch failed to maintain format integrity
                    std::vector<uint8_t> zeros(m.size_bytes, 0);
                    oss.write(reinterpret_cast<const char*>(zeros.data()),
                              static_cast<std::streamsize>(zeros.size()));
                    std::cerr << "[cluster] export: failed to fetch media " << m.id << std::endl;
                }
            }
        }
    }
    auto str = oss.str();
    return std::vector<std::uint8_t>(str.begin(), str.end());
}

bool ClusterMediaBackend::import_db(const std::string& path) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!local_.import_db(path)) return false;
    // Replicate each media's raw data individually to cluster
    for (const auto& sid : local_.store_ids()) {
        for (const auto& album : local_.list_albums(sid)) {
            for (const auto& m : local_.list_media(album.id)) {
                auto raw = local_.read_media_data(m.id);
                if (!raw.empty()) {
                    replicate_media(m.id, raw.data(), raw.size());
                }
            }
        }
        replicate_store(sid);
    }
    replicate_index();
    return true;
}

bool ClusterMediaBackend::import_db_from_buffer(const std::vector<std::uint8_t>& data) {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!cluster_.isRunning()) {
        std::cerr << "[cluster] import rejected: cluster not running" << std::endl;
        return false;
    }
    bool replicate_ok = true;
    std::vector<std::string> failed_keys;
    // First import into local BinDb (full data) to extract individual media,
    // then replicate each media separately to the cluster
    auto replicate_fn = [this, &replicate_ok, &failed_keys](const std::string& key, const uint8_t* d, size_t len) {
        std::cerr << "[cluster] import replicate(" << key << "): " << len << " bytes" << std::endl;
        if (!cluster_.replicate(key, d, len)) {
            std::cerr << "[cluster] import replicate failed: " << key << std::endl;
            replicate_ok = false;
            failed_keys.push_back(key);
        }
    };
    bool parse_ok = local_.import_db_from_buffer(data, replicate_fn);
    if (!parse_ok) return false;

    // Now replicate each media's raw data individually and remove from local memory
    for (const auto& sid : local_.store_ids()) {
        for (const auto& album : local_.list_albums(sid)) {
            for (const auto& m : local_.list_media(album.id)) {
                auto raw = local_.read_media_data(m.id);
                if (!raw.empty()) {
                    std::string mkey = "media:" + m.id;
                    std::cerr << "[cluster] import replicate(" << mkey << "): " << raw.size() << " bytes" << std::endl;
                    if (!cluster_.replicate(mkey, raw.data(), raw.size())) {
                        std::cerr << "[cluster] import replicate failed: " << mkey << std::endl;
                        failed_keys.push_back(mkey);
                    }
                }
            }
        }
        // Replicate metadata-only store
        replicate_store(sid);
    }
    replicate_index();

    if (!replicate_ok || !failed_keys.empty()) {
        constexpr int MAX_RETRIES = 3;
        for (int attempt = 1; attempt <= MAX_RETRIES && !failed_keys.empty(); ++attempt) {
            std::cerr << "[cluster] import: " << failed_keys.size()
                      << " key(s) failed, retry " << attempt
                      << "/" << MAX_RETRIES << std::endl;
            std::this_thread::sleep_for(std::chrono::seconds(attempt * 2));
            std::vector<std::string> still_failed;
            for (const auto& key : failed_keys) {
                bool ok = false;
                if (key == "index") {
                    auto idx_buf = local_.save_index_to_buffer();
                    ok = cluster_.replicate("index", idx_buf.data(), idx_buf.size());
                } else if (key.starts_with("store:")) {
                    std::string sid = key.substr(6);
                    auto store_buf = local_.save_store_metadata_to_buffer(sid);
                    ok = cluster_.replicate(key, store_buf.data(), store_buf.size());
                } else if (key.starts_with("media:")) {
                    std::string mid = key.substr(6);
                    auto raw = local_.read_media_data(mid);
                    ok = cluster_.replicate(key, raw.data(), raw.size());
                }
                if (!ok) still_failed.push_back(key);
            }
            failed_keys = std::move(still_failed);
            if (failed_keys.empty()) {
                std::cerr << "[cluster] import: all retries succeeded on attempt " << attempt << std::endl;
                break;
            }
        }
        if (!failed_keys.empty()) {
            std::cerr << "[cluster] import: " << failed_keys.size()
                      << " key(s) still failed after retries:";
            for (const auto& k : failed_keys) std::cerr << " " << k;
            std::cerr << std::endl;
        }
    }
    return parse_ok;
}

void ClusterMediaBackend::vacuum_store(const std::string& store_id) {
    local_.vacuum_store(store_id);
}

void ClusterMediaBackend::vacuum_all_stores() {
    local_.vacuum_all_stores();
}

// ---- cluster sync ----
void ClusterMediaBackend::sync_from_cluster() {
    std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
    if (!cluster_.isRunning()) return;

    // Fetch index from peers
    std::vector<uint8_t> index_data;
    if (cluster_.fetch("index", index_data) && !index_data.empty()) {
        local_.load_index_from_buffer(index_data);
    }

    // Fetch metadata-only store data from peers (no raw media data)
    for (const auto& sid : local_.store_ids()) {
        std::vector<uint8_t> store_data;
        if (cluster_.fetch("store:" + sid, store_data) && !store_data.empty()) {
            local_.load_store_metadata_from_buffer(sid, store_data);
        }
    }
}

BinDb& ClusterMediaBackend::local() { return local_; }

void ClusterMediaBackend::replicate_index() {
    if (!cluster_.isRunning()) return;
    auto buf = local_.save_index_to_buffer();
    if (buf.empty()) {
        std::cerr << "[cluster] replicate_index: buffer empty, skipping" << std::endl;
        return;
    }
    std::cerr << "[cluster] replicate_index: " << buf.size() << " bytes, stores=" << local_.store_ids().size() << std::endl;
    cluster_.replicate("index", buf.data(), buf.size());
}

void ClusterMediaBackend::replicate_store(const std::string& store_id) {
    if (!cluster_.isRunning()) return;
    // Replicate metadata only — no raw media data in store buffer
    auto buf = local_.save_store_metadata_to_buffer(store_id);
    if (buf.empty()) {
        std::cerr << "[cluster] replicate_store(" << store_id << "): buffer empty, skipping" << std::endl;
        return;
    }
    std::cerr << "[cluster] replicate_store(" << store_id << "): " << buf.size() << " bytes (metadata only)" << std::endl;
    cluster_.replicate("store:" + store_id, buf.data(), buf.size());
}

void ClusterMediaBackend::replicate_media(const std::string& media_id, const uint8_t* data, size_t len) {
    if (!cluster_.isRunning()) return;
    std::string key = "media:" + media_id;
    std::cerr << "[cluster] replicate_media(" << media_id << "): " << len << " bytes" << std::endl;
    cluster_.replicate(key, data, len);
}

bool ClusterMediaBackend::fetch_media_from_cluster(const std::string& media_id, std::vector<uint8_t>& out) const {
    std::string key = "media:" + media_id;
    if (cluster_.isRunning()) {
        if (cluster_.fetch(key, out) && !out.empty()) {
            return true;
        }
    }
    std::cerr << "[cluster] fetch_media(" << media_id << "): failed" << std::endl;
    return false;
}

void ClusterMediaBackend::repair_replication() {
    if (!cluster_.isRunning()) return;
    size_t total_peers = cluster_.getTotalPeers();
    if (total_peers < 2) return;

    const auto& cfg = cluster_.getConfig();
    size_t required_shards = cfg.data_blocks + cfg.parity_blocks;

    auto peer_groups = cluster_.list_peer_groups();
    std::vector<std::unordered_set<uint64_t>> node_group_sets(peer_groups.size());
    for (size_t i = 0; i < peer_groups.size(); ++i) {
        node_group_sets[i].insert(peer_groups[i].groups.begin(),
                                  peer_groups[i].groups.end());
    }

    size_t online_count = 0;
    for (const auto& pg : peer_groups)
        if (pg.online) ++online_count;
    if (online_count < required_shards) return; // not enough nodes online to repair

    // Check index
    uint64_t index_gid = cluster_group_id("index");
    int index_shards = 0;
    for (size_t i = 0; i < peer_groups.size(); ++i) {
        if (peer_groups[i].online && node_group_sets[i].count(index_gid))
            ++index_shards;
    }
    if (index_shards < static_cast<int>(required_shards)) {
        std::cerr << "[cluster] repair: index degraded (" << index_shards
                  << "/" << required_shards << " shards), re-replicating" << std::endl;
        replicate_index();
    }

    // Check stores and their media
    for (const auto& sid : local_.store_ids()) {
        uint64_t gid = cluster_group_id("store:" + sid);
        int shards = 0;
        for (size_t i = 0; i < peer_groups.size(); ++i) {
            if (peer_groups[i].online && node_group_sets[i].count(gid))
                ++shards;
        }
        if (shards < static_cast<int>(required_shards)) {
            std::cerr << "[cluster] repair: store " << sid << " degraded ("
                      << shards << "/" << required_shards << " shards), re-replicating" << std::endl;
            replicate_store(sid);
        }

        // Check individual media data in this store
        for (const auto& album : local_.list_albums(sid)) {
            for (const auto& m : local_.list_media(album.id)) {
                std::string mkey = "media:" + m.id;
                uint64_t mgid = cluster_group_id(mkey);
                int mshards = 0;
                for (size_t i = 0; i < peer_groups.size(); ++i) {
                    if (peer_groups[i].online && node_group_sets[i].count(mgid))
                        ++mshards;
                }
                if (mshards < static_cast<int>(required_shards)) {
                    // Fetch from available shards and re-replicate
                    std::vector<uint8_t> raw;
                    if (fetch_media_from_cluster(m.id, raw) && !raw.empty()) {
                        std::cerr << "[cluster] repair: media " << m.id << " degraded ("
                                  << mshards << "/" << required_shards << " shards), re-replicating" << std::endl;
                        replicate_media(m.id, raw.data(), raw.size());
                    }
                }
            }
        }
    }
}

void ClusterMediaBackend::sync_loop() {
    while (sync_running_) {
        {
            std::unique_lock<std::mutex> lock(sync_mutex_);
            sync_cv_.wait_for(lock, std::chrono::seconds(10),
                              [this]() { return !sync_running_.load(); });
        }
        if (!sync_running_) break;
        try {
            sync_from_cluster();
            repair_replication();
        } catch (const std::exception& e) {
            std::cerr << "Cluster sync error: " << e.what() << std::endl;
        }
    }
}

} // namespace mediadb
