mdl_cache_io.c

Go to the documentation of this file.

#include <limits.h>
#include <stdint.h>
#include <string.h>
 
#include "mdl_cache_internal.h"
#include "mdl_hash.h"
#include "mdl_sanitize.h"
#include "mdl_url_guard.h"

typedef enum : uint32_t {
  k_cache_header_bytes    = 32U,      
  k_cache_record_bytes    = 36U,      
  k_cache_trailer_bytes   = 8U,       
  k_cache_io_call_max     = 2000000U, 
  k_cache_status_min      = 100U,     
  k_cache_status_max      = 599U,     
  k_cache_hex_high_shift  = 60U,      
  k_cache_nibble_mask     = 0x0FU,    
  k_cache_u64_high_shift  = 56U,      
  k_cache_record_status   = 24U,      
  k_cache_record_url_len  = 26U,      
  k_cache_record_path_len = 28U,      
  k_cache_record_etag_len = 30U,      
  k_cache_record_time_len = 32U,      
  k_cache_record_reserved = 34U,      
  k_cache_header_count    = 12U,      
  k_cache_header_host     = 24U,      
} mdl_cache_binary_limit_t;

static const uint8_t s_cache_magic[k_cache_trailer_bytes] =
  {'M', 'D', 'L', 'C', 'A', 'C', 'H', '1'};

RA8_INTERNAL static void internal_cache_hex16(char* destination, uint64_t value)
{
  static const char k_hex[] = "0123456789abcdef";
  for (uint8_t i = 0U; i < 16U; ++i) {
    const uint8_t shift = (uint8_t)(k_cache_hex_high_shift - (4U * i));
    destination[i]      = k_hex[(value >> shift) & k_cache_nibble_mask];
  }
}

RA8_INTERNAL static bool
internal_cache_host_leaf(char* destination, size_t capacity, uint64_t host_hash)
{
  static const char k_prefix[] = "host-";
  const size_t      required   = (sizeof(k_prefix) - 1U) + 16U + 1U;
  if (capacity < required) {
    return false;
  }
  memcpy(destination, k_prefix, sizeof(k_prefix) - 1U);
  internal_cache_hex16(&destination[sizeof(k_prefix) - 1U], host_hash);
  destination[required - 1U] = '\0';
  return true;
}

RA8_INTERNAL static bool internal_cache_body_leaf(char*    destination,
                                                  size_t   capacity,
                                                  uint64_t url_hash,
                                                  uint64_t content_hash)
{
  static const char k_prefix[] = "body-";
  static const char k_suffix[] = ".cache";
  const size_t      required   = (sizeof(k_prefix) - 1U) + 16U + 1U + 16U + sizeof(k_suffix);
  if (capacity < required) {
    return false;
  }
  size_t cursor = 0U;
  memcpy(&destination[cursor], k_prefix, sizeof(k_prefix) - 1U);
  cursor += sizeof(k_prefix) - 1U;
  internal_cache_hex16(&destination[cursor], url_hash);
  cursor += 16U;
  destination[cursor++] = '-';
  internal_cache_hex16(&destination[cursor], content_hash);
  cursor += 16U;
  memcpy(&destination[cursor], k_suffix, sizeof(k_suffix));
  return true;
}

RA8_INTERNAL static void internal_cache_put_u16(uint8_t* out, uint16_t value)
{
  out[0] = (uint8_t)(value >> 8U);
  out[1] = (uint8_t)value;
}

RA8_INTERNAL static void internal_cache_put_u64(uint8_t* out, uint64_t value)
{
  for (uint8_t i = 0U; i < 8U; ++i) {
    out[i] = (uint8_t)(value >> (k_cache_u64_high_shift - (8U * i)));
  }
}

RA8_INTERNAL static uint16_t internal_cache_get_u16(const uint8_t* in)
{
  return (uint16_t)(((uint16_t)in[0] << 8U) | (uint16_t)in[1]);
}

RA8_INTERNAL static uint64_t internal_cache_get_u64(const uint8_t* in)
{
  uint64_t value = 0U;
  for (uint8_t i = 0U; i < 8U; ++i) {
    value = (value << 8U) | (uint64_t)in[i];
  }
  return value;
}

RA8_INTERNAL static ra8_err_t
internal_cache_read_all(fw_fs_file_t* file, uint8_t* destination, uint32_t length, uint32_t* calls)
{
  uint32_t offset = 0U;
  while (offset < length) {
    if (*calls >= (uint32_t)k_cache_io_call_max) {
      return k_ra8_err_invalid_size;
    }
    uint32_t        received = 0U;
    const ra8_err_t error    = fw_fs_read(file, destination + offset, length - offset, &received);
    ++(*calls);
    if (error != k_ra8_ok) {
      return error;
    }
    if (received == 0U) {
      return k_ra8_err_invalid_state;
    }
    offset += received;
  }
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t internal_cache_ensure_directory(mdl_storage_t* storage,
                                                              const char*    path)
{
  fw_fs_stat_t node  = {};
  ra8_err_t    error = fw_fs_stat(&storage->fs->names, path, &node);
  if (error != k_ra8_ok) {
    return error;
  }
  if (!node.exists) {
    error = fw_fs_mkdir(&storage->fs->names, path);
    if (error != k_ra8_ok) {
      return error;
    }
    node  = (fw_fs_stat_t){};
    error = fw_fs_stat(&storage->fs->names, path, &node);
  }
  return ((error == k_ra8_ok) && node.exists && (node.type == k_fw_fs_node_directory))
           ? k_ra8_ok
           : k_ra8_err_invalid_state;
}

RA8_INTERNAL static ra8_err_t
internal_cache_paths(mdl_cache_t* cache, const char* url, mdl_cache_paths_t* paths)
{
  *paths = (mdl_cache_paths_t){};
  if ((cache == nullptr) || (cache->storage == nullptr) || (cache->root == nullptr) ||
      (cache->root[0] != '/') || !mdl_url_host(url, paths->host, sizeof(paths->host))) {
    return k_ra8_err_invalid_arg;
  }
  paths->host_hash = mdl_hash_str(paths->host);
  char leaf[32];
  if (!internal_cache_host_leaf(leaf, sizeof(leaf), paths->host_hash) ||
      !mdl_path_join(cache->root, leaf, paths->directory, sizeof(paths->directory)) ||
      !mdl_path_join(paths->directory, "index.v1", paths->index_path, sizeof(paths->index_path))) {
    return k_ra8_err_invalid_size;
  }
  ra8_err_t error = internal_cache_ensure_directory(cache->storage, cache->root);
  return (error == k_ra8_ok) ? internal_cache_ensure_directory(cache->storage, paths->directory)
                             : error;
}

RA8_INTERNAL static bool internal_cache_record_valid(const mdl_cache_record_t* record,
                                                     const mdl_cache_paths_t*  paths)
{
  char       host[k_mdl_gov_host_max];
  const bool fields = (record->url[0] != '\0') && (record->relative_path[0] != '\0') &&
                      (strpbrk(record->url, "\t\r\n") == nullptr) &&
                      (strpbrk(record->relative_path, "/\\\t\r\n") == nullptr) &&
                      (strpbrk(record->etag, "\r\n") == nullptr) &&
                      (strpbrk(record->last_modified, "\r\n") == nullptr) &&
                      (record->fetched_at >= 0) &&
                      (record->response_status >= (uint16_t)k_cache_status_min) &&
                      (record->response_status <= (uint16_t)k_cache_status_max);
  return fields && (record->url_hash == mdl_hash_str(record->url)) &&
         mdl_url_host(record->url, host, sizeof(host)) && (mdl_hash_str(host) == paths->host_hash);
}

RA8_INTERNAL static ra8_err_t internal_cache_read_record(fw_fs_file_t*            file,
                                                         uint32_t*                calls,
                                                         uint64_t*                hash,
                                                         mdl_cache_record_t*      record,
                                                         const mdl_cache_paths_t* paths)
{
  uint8_t   header[k_cache_record_bytes];
  ra8_err_t error = internal_cache_read_all(file, header, sizeof(header), calls);
  if (error != k_ra8_ok) {
    return error;
  }
  *hash                       = mdl_hash_bytes_seed(header, sizeof(header), *hash);
  const uint16_t lengths[]    = {internal_cache_get_u16(&header[k_cache_record_url_len]),
                                 internal_cache_get_u16(&header[k_cache_record_path_len]),
                                 internal_cache_get_u16(&header[k_cache_record_etag_len]),
                                 internal_cache_get_u16(&header[k_cache_record_time_len])};
  const size_t   capacities[] = {sizeof(record->url),
                                 sizeof(record->relative_path),
                                 sizeof(record->etag),
                                 sizeof(record->last_modified)};
  char* fields[] = {record->url, record->relative_path, record->etag, record->last_modified};
  if ((internal_cache_get_u16(&header[k_cache_record_reserved]) != 0U) || (lengths[0] == 0U)) {
    return k_ra8_err_invalid_state;
  }
  *record =
    (mdl_cache_record_t){.url_hash        = internal_cache_get_u64(&header[0]),
                         .content_hash    = internal_cache_get_u64(&header[8]),
                         .fetched_at      = (int64_t)internal_cache_get_u64(&header[16]),
                         .response_status = internal_cache_get_u16(&header[k_cache_record_status])};
  for (size_t i = 0U; i < 4U; ++i) {
    if ((size_t)lengths[i] >= capacities[i]) {
      return k_ra8_err_invalid_size;
    }
    error = internal_cache_read_all(file, (uint8_t*)fields[i], lengths[i], calls);
    if (error != k_ra8_ok) {
      return error;
    }
    fields[i][lengths[i]] = '\0';
    *hash                 = mdl_hash_bytes_seed(fields[i], lengths[i], *hash);
  }
  return internal_cache_record_valid(record, paths) ? k_ra8_ok : k_ra8_err_invalid_state;
}

RA8_INTERNAL static ra8_err_t
internal_cache_decode(mdl_cache_t* cache, const mdl_cache_paths_t* paths, uint64_t size_bytes)
{
  fw_fs_file_t file                         = {};
  ra8_err_t    error                        = fw_fs_open(&cache->storage->fs->streams,
                                                         paths->index_path,
                                                         k_fw_fs_open_read,
                                                         &file,
                                                         cache->storage->file_workspace,
                                                         cache->storage->file_workspace_bytes);
  uint32_t     calls                        = 0U;
  uint8_t      header[k_cache_header_bytes] = {};
  if (error == k_ra8_ok) {
    error = internal_cache_read_all(&file, header, sizeof(header), &calls);
  }
  uint16_t count = 0U;
  if (error == k_ra8_ok) {
    count                        = internal_cache_get_u16(&header[k_cache_header_count]);
    const uint64_t payload_bytes = internal_cache_get_u64(&header[16]);
    const uint64_t expected_size =
      (uint64_t)k_cache_header_bytes + payload_bytes + (uint64_t)k_cache_trailer_bytes;
    if ((memcmp(header, s_cache_magic, sizeof(s_cache_magic)) != 0) ||
        (internal_cache_get_u16(&header[8]) != (uint16_t)k_mdl_cache_schema_version) ||
        (internal_cache_get_u16(&header[10]) != (uint16_t)k_cache_header_bytes) ||
        (count > (uint16_t)k_mdl_cache_record_max) || (internal_cache_get_u16(&header[14]) != 0U) ||
        (expected_size != size_bytes) ||
        (internal_cache_get_u64(&header[k_cache_header_host]) != paths->host_hash)) {
      error = k_ra8_err_invalid_state;
    }
  }
  *cache->index = (mdl_cache_index_t){.host_hash      = paths->host_hash,
                                      .schema_version = k_mdl_cache_schema_version};
  uint64_t hash = (uint64_t)k_mdl_fnv_offset;
  for (uint16_t i = 0U; (error == k_ra8_ok) && (i < count); ++i) {
    error = internal_cache_read_record(&file, &calls, &hash, &cache->index->records[i], paths);
    cache->index->record_count = (uint16_t)(i + 1U);
  }
  uint8_t trailer[k_cache_trailer_bytes];
  if (error == k_ra8_ok) {
    error = internal_cache_read_all(&file, trailer, sizeof(trailer), &calls);
  }
  if ((error == k_ra8_ok) && (internal_cache_get_u64(trailer) != hash)) {
    error = k_ra8_err_invalid_state;
  }
  const ra8_err_t closed = file.is_open ? fw_fs_close(&file) : k_ra8_ok;
  return (error == k_ra8_ok) ? closed : error;
}

RA8_INTERNAL static ra8_err_t internal_cache_discard_index(mdl_cache_t* cache, const char* path)
{
  fw_fs_stat_t    node  = {};
  const ra8_err_t error = fw_fs_stat(&cache->storage->fs->names, path, &node);
  if (error != k_ra8_ok) {
    return error;
  }
  if (!node.exists) {
    return k_ra8_ok;
  }
  if (node.type != k_fw_fs_node_file) {
    return k_ra8_err_invalid_state;
  }
  return fw_fs_unlink(&cache->storage->fs->names, path);
}
 
RA8_PRIV ra8_err_t priv_mdl_cache_load(mdl_cache_t*       cache,
                                       const char*        url,
                                       mdl_cache_paths_t* paths,
                                       bool*              rebuilt)
{
  if ((cache == nullptr) || (cache->storage == nullptr) || (cache->index == nullptr) ||
      (url == nullptr) || (paths == nullptr) || (rebuilt == nullptr)) {
    return k_ra8_err_invalid_arg;
  }
  *rebuilt        = false;
  ra8_err_t error = internal_cache_paths(cache, url, paths);
  if (error != k_ra8_ok) {
    return error;
  }
  fw_fs_stat_t node = {};
  error             = fw_fs_stat(&cache->storage->fs->names, paths->index_path, &node);
  if (error != k_ra8_ok) {
    return error;
  }
  if (!node.exists) {
    *cache->index = (mdl_cache_index_t){.host_hash      = paths->host_hash,
                                        .schema_version = k_mdl_cache_schema_version};
    return k_ra8_ok;
  }
  if (node.type != k_fw_fs_node_file) {
    return k_ra8_err_invalid_state;
  }
  error = internal_cache_decode(cache, paths, node.size_bytes);
  if (error == k_ra8_ok) {
    return k_ra8_ok;
  }
  error = internal_cache_discard_index(cache, paths->index_path);
  if (error != k_ra8_ok) {
    return error;
  }
  *cache->index = (mdl_cache_index_t){.host_hash      = paths->host_hash,
                                      .schema_version = k_mdl_cache_schema_version};
  *rebuilt      = true;
  return k_ra8_ok;
}

RA8_INTERNAL static bool
internal_cache_encode_record(const mdl_cache_record_t* record, uint8_t* header, uint16_t* lengths)
{
  const char* fields[] = {record->url, record->relative_path, record->etag, record->last_modified};
  for (size_t i = 0U; i < 4U; ++i) {
    const size_t length = strlen(fields[i]);
    if (length > UINT16_MAX) {
      return false;
    }
    lengths[i] = (uint16_t)length;
  }
  memset(header, 0, k_cache_record_bytes);
  internal_cache_put_u64(&header[0], record->url_hash);
  internal_cache_put_u64(&header[8], record->content_hash);
  internal_cache_put_u64(&header[16], (uint64_t)record->fetched_at);
  internal_cache_put_u16(&header[k_cache_record_status], record->response_status);
  internal_cache_put_u16(&header[k_cache_record_url_len], lengths[0]);
  internal_cache_put_u16(&header[k_cache_record_path_len], lengths[1]);
  internal_cache_put_u16(&header[k_cache_record_etag_len], lengths[2]);
  internal_cache_put_u16(&header[k_cache_record_time_len], lengths[3]);
  return true;
}

RA8_INTERNAL static ra8_err_t internal_cache_payload_identity(const mdl_cache_index_t* index,
                                                              uint64_t*                out_bytes,
                                                              uint64_t*                out_hash)
{
  uint64_t bytes = 0U;
  uint64_t hash  = (uint64_t)k_mdl_fnv_offset;
  for (uint16_t i = 0U; i < index->record_count; ++i) {
    uint8_t  header[k_cache_record_bytes];
    uint16_t lengths[4];
    if (!internal_cache_encode_record(&index->records[i], header, lengths)) {
      return k_ra8_err_invalid_size;
    }
    hash = mdl_hash_bytes_seed(header, sizeof(header), hash);
    bytes += sizeof(header);
    const char* fields[] = {index->records[i].url,
                            index->records[i].relative_path,
                            index->records[i].etag,
                            index->records[i].last_modified};
    for (size_t field = 0U; field < 4U; ++field) {
      hash = mdl_hash_bytes_seed(fields[field], lengths[field], hash);
      bytes += lengths[field];
    }
  }
  *out_bytes = bytes;
  *out_hash  = hash;
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t internal_cache_write_records(mdl_storage_txn_t*       writer,
                                                           const mdl_cache_index_t* index)
{
  for (uint16_t i = 0U; i < index->record_count; ++i) {
    uint8_t  header[k_cache_record_bytes];
    uint16_t lengths[4];
    if (!internal_cache_encode_record(&index->records[i], header, lengths)) {
      return k_ra8_err_invalid_size;
    }
    ra8_err_t   error    = mdl_storage_txn_write(writer, header, sizeof(header));
    const char* fields[] = {index->records[i].url,
                            index->records[i].relative_path,
                            index->records[i].etag,
                            index->records[i].last_modified};
    for (size_t field = 0U; (error == k_ra8_ok) && (field < 4U); ++field) {
      error = mdl_storage_txn_write(writer, (const uint8_t*)fields[field], lengths[field]);
    }
    if (error != k_ra8_ok) {
      return error;
    }
  }
  return k_ra8_ok;
}
 
RA8_PRIV ra8_err_t priv_mdl_cache_save(mdl_cache_t* cache, const mdl_cache_paths_t* paths)
{
  if ((cache == nullptr) || (cache->storage == nullptr) || (cache->index == nullptr) ||
      (paths == nullptr) || (cache->index->schema_version != k_mdl_cache_schema_version) ||
      (cache->index->host_hash != paths->host_hash) ||
      (cache->index->record_count > (uint16_t)k_mdl_cache_record_max)) {
    return k_ra8_err_invalid_arg;
  }
  for (uint16_t i = 0U; i < cache->index->record_count; ++i) {
    if (!internal_cache_record_valid(&cache->index->records[i], paths)) {
      return k_ra8_err_invalid_state;
    }
  }
  uint64_t  payload_bytes = 0U;
  uint64_t  payload_hash  = 0U;
  ra8_err_t error = internal_cache_payload_identity(cache->index, &payload_bytes, &payload_hash);
  uint8_t   header[k_cache_header_bytes] = {};
  memcpy(header, s_cache_magic, sizeof(s_cache_magic));
  internal_cache_put_u16(&header[8], (uint16_t)k_mdl_cache_schema_version);
  internal_cache_put_u16(&header[10], (uint16_t)k_cache_header_bytes);
  internal_cache_put_u16(&header[k_cache_header_count], cache->index->record_count);
  internal_cache_put_u64(&header[16], payload_bytes);
  internal_cache_put_u64(&header[k_cache_header_host], paths->host_hash);
  uint8_t trailer[k_cache_trailer_bytes];
  internal_cache_put_u64(trailer, payload_hash);
  mdl_storage_txn_t writer = {};
  if (error == k_ra8_ok) {
    error = mdl_storage_txn_begin(&writer, cache->storage, paths->index_path);
  }
  if (error == k_ra8_ok) {
    error = mdl_storage_txn_write(&writer, header, sizeof(header));
  }
  if (error == k_ra8_ok) {
    error = internal_cache_write_records(&writer, cache->index);
  }
  if (error == k_ra8_ok) {
    error = mdl_storage_txn_write(&writer, trailer, sizeof(trailer));
  }
  if (error == k_ra8_ok) {
    return mdl_storage_txn_commit(&writer);
  }
  const ra8_err_t aborted = mdl_storage_txn_abort(&writer);
  return (aborted == k_ra8_ok) ? error : aborted;
}

RA8_INTERNAL static ra8_err_t internal_cache_read_body_exact(mdl_storage_t* storage,
                                                             const char*    body_path,
                                                             char*          buffer,
                                                             uint64_t       size_bytes)
{
  fw_fs_file_t file  = {};
  ra8_err_t    error = fw_fs_open(&storage->fs->streams,
                                  body_path,
                                  k_fw_fs_open_read,
                                  &file,
                                  storage->file_workspace,
                                  storage->file_workspace_bytes);
  uint32_t     calls = 0U;
  if (error == k_ra8_ok) {
    error = internal_cache_read_all(&file, (uint8_t*)buffer, (uint32_t)size_bytes, &calls);
  }
  uint8_t  extra = 0U;
  uint32_t got   = 0U;
  if (error == k_ra8_ok) {
    error = fw_fs_read(&file, &extra, 1U, &got);
  }
  if ((error == k_ra8_ok) && (got != 0U)) {
    error = k_ra8_err_invalid_state;
  }
  const ra8_err_t closed = file.is_open ? fw_fs_close(&file) : k_ra8_ok;
  if (error == k_ra8_ok) {
    error = closed;
  }
  return error;
}
 
RA8_PRIV ra8_err_t priv_mdl_cache_read_body(mdl_storage_t*            storage,
                                            const mdl_cache_paths_t*  paths,
                                            const mdl_cache_record_t* record,
                                            char*                     buffer,
                                            size_t                    capacity,
                                            size_t*                   out_length)
{
  if ((storage == nullptr) || (paths == nullptr) || (record == nullptr) || (buffer == nullptr) ||
      (capacity == 0U) || (out_length == nullptr)) {
    return k_ra8_err_invalid_arg;
  }
  *out_length = 0U;
  char body_path[k_fw_fs_path_cap];
  if (!mdl_path_join(paths->directory, record->relative_path, body_path, sizeof(body_path))) {
    return k_ra8_err_invalid_size;
  }
  fw_fs_stat_t node  = {};
  ra8_err_t    error = fw_fs_stat(&storage->fs->names, body_path, &node);
  if ((error != k_ra8_ok) || !node.exists || (node.type != k_fw_fs_node_file)) {
    return (error != k_ra8_ok) ? error : k_ra8_err_not_found;
  }
  if ((node.size_bytes == 0U) || (node.size_bytes > capacity) || (node.size_bytes > UINT32_MAX)) {
    return k_ra8_err_invalid_size;
  }
  error = internal_cache_read_body_exact(storage, body_path, buffer, node.size_bytes);
  if ((error == k_ra8_ok) &&
      (mdl_hash_bytes(buffer, (size_t)node.size_bytes) != record->content_hash)) {
    error = k_ra8_err_validation_failed;
  }
  if (error == k_ra8_ok) {
    *out_length = (size_t)node.size_bytes;
  }
  return error;
}
 
RA8_PRIV ra8_err_t priv_mdl_cache_publish_body(mdl_storage_t*           storage,
                                               const mdl_cache_paths_t* paths,
                                               uint64_t                 url_hash,
                                               uint64_t                 content_hash,
                                               const char*              buffer,
                                               size_t                   length,
                                               char*                    relative_path,
                                               size_t                   relative_capacity)
{
  if ((storage == nullptr) || (paths == nullptr) || (buffer == nullptr) || (length == 0U) ||
      (length > UINT32_MAX) || (relative_path == nullptr) || (relative_capacity == 0U)) {
    return k_ra8_err_invalid_arg;
  }
  char body_path[k_fw_fs_path_cap];
  if (!internal_cache_body_leaf(relative_path, relative_capacity, url_hash, content_hash) ||
      !mdl_path_join(paths->directory, relative_path, body_path, sizeof(body_path))) {
    return k_ra8_err_invalid_size;
  }
  mdl_storage_txn_t writer = {};
  ra8_err_t         error  = mdl_storage_txn_begin(&writer, storage, body_path);
  if (error == k_ra8_ok) {
    error = mdl_storage_txn_write(&writer, (const uint8_t*)buffer, (uint32_t)length);
  }
  if (error == k_ra8_ok) {
    return mdl_storage_txn_commit(&writer);
  }
  const ra8_err_t aborted = mdl_storage_txn_abort(&writer);
  return (aborted == k_ra8_ok) ? error : aborted;
}