mdl_fetch.c

Go to the documentation of this file.

#include "mdl_fetch.h"
 
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
 
#include "mdl_fetch_body_internal.h"
#include "mdl_fetch_internal.h"
#include "mdl_net.h"
#include "mdl_pathfs.h"
#include "mdl_storage.h"
#include "mdl_stream_internal.h"
#include "mdl_url_guard.h"
#include "mdl_urlname.h"
#include "ra8_attributes.h"
#include "ra8_err.h"

typedef enum : uint16_t {
  k_http_server_err = 500, 
} mdl_fetch_http_t;

typedef enum : uint8_t {
  k_fetch_max_attempts = 4U, 
} mdl_fetch_retry_t;

typedef enum : uint8_t {
  k_ch_completed = 0, 
  k_ch_skipped   = 1, 
  k_ch_failed    = 2, 
} mdl_chap_status_t;

typedef struct {
  char               id[k_mdl_chapter_id_max];         
  char               extracted_title[k_mdl_title_max]; 
  double             number;                           
  bool               number_known;                     
  size_t             html_len;                         
  const char*        title;                            
  mdl_chapter_rec_t* rec;                              
} mdl_chapter_work_t;

RA8_INTERNAL static ra8_err_t internal_mdl_fetch_diag3(mdl_fetch_ctx_t* ctx,
                                                       const char*      first,
                                                       const char*      second,
                                                       const char*      third)
{
  ra8_err_t error = priv_mdl_stream_text(k_ra8_ok, ctx->diagnostic, first);
  error           = priv_mdl_stream_text(error, ctx->diagnostic, second);
  error           = priv_mdl_stream_text(error, ctx->diagnostic, third);
  if (error != k_ra8_ok) {
    ctx->progress_error = error;
  }
  return error;
}

RA8_INTERNAL static uint32_t internal_mdl_fetch_max_u32(uint32_t a, uint32_t b)
{
  return (a > b) ? a : b;
}
 
RA8_PRIV bool priv_mdl_fetch_is_retryable(ra8_err_t rc)
{
  return (rc == k_ra8_err_busy) || (rc == k_ra8_err_timeout) || (rc == k_ra8_fail);
}
 
RA8_PRIV bool priv_mdl_fetch_run_incomplete(const mdl_fetch_stats_t* stats)
{
  if (stats == nullptr) {
    return false;
  }
  return (stats->chapters_failed > 0U) || (stats->pages_failed > 0U);
}

RA8_INTERNAL static const char* internal_mdl_fetch_fail_reason(long status)
{
  return (status >= (long)k_http_server_err) ? "server error" : "transport error";
}
 
RA8_PRIV void priv_mdl_fetch_reason(ra8_err_t err, long status, char* buf, size_t cap)
{
  if (buf == nullptr) {
    return;
  }
  if (cap == 0U) {
    return;
  }
  const char* base = nullptr;
  switch (err) {
    case k_ra8_ok:
      base = "ok";
      break;
    case k_ra8_err_timeout:
      base = "request timed out";
      break;
    case k_ra8_err_busy:
      base = "rate limited";
      break;
    case k_ra8_err_not_found:
      base = "not found";
      break;
    case k_ra8_err_no_mem:
      base = "response exceeded size cap";
      break;
    case k_ra8_err_retry_limit:
      base = "still failing after retries";
      break;
    case k_ra8_fail:
      base = internal_mdl_fetch_fail_reason(status);
      break;
    default:
      base = "error";
      break;
  }
  if (status > 0) {
    (void)snprintf(buf, cap, "%s (HTTP %ld)", base, status);
  } else {
    (void)snprintf(buf, cap, "%s", base);
  }
}

RA8_PRIV void
priv_mdl_fetch_record_fail(const mdl_fetch_ctx_t* ctx, const char* url, long status, ra8_err_t err)
{
  if (ctx->faillog == nullptr) {
    return;
  }
  mdl_fetch_faillog_t* log = ctx->faillog;
  log->total += 1U;
  if (log->count >= (size_t)k_mdl_fetch_fail_max) {
    return;
  }
  mdl_fetch_fail_t* item = &log->items[log->count];
  (void)snprintf(item->url, sizeof(item->url), "%s", (url != nullptr) ? url : "");
  item->status = status;
  item->err    = err;
  log->count += 1U;
}

RA8_PRIV ra8_err_t priv_mdl_fetch_checkpoint(const mdl_fetch_ctx_t* ctx)
{
  if (ctx->state_path == nullptr) {
    return k_ra8_ok;
  }
  bool            published = false;
  const ra8_err_t rc        = mdl_state_save(ctx->storage, ctx->state_path, ctx->state, &published);
  if (rc != k_ra8_ok) {
    priv_mdl_fetch_record_fail(ctx, ctx->state_path, 0L, rc);
  }
  return rc;
}

RA8_INTERNAL static const char* internal_mdl_fetch_page_host(const char* url, char* buf, size_t cap)
{
  return mdl_url_host(url, buf, cap) ? buf : nullptr;
}

RA8_INTERNAL static ra8_err_t internal_mdl_fetch_governed_get_body(mdl_fetch_ctx_t*     ctx,
                                                                   const char*          host,
                                                                   const char*          url,
                                                                   const mdl_net_req_t* req,
                                                                   mdl_net_body_sink_t* sink,
                                                                   uint32_t             jmin,
                                                                   uint32_t             jmax,
                                                                   mdl_net_resp_t*      out_resp,
                                                                   size_t*              out_bytes)
{
  if (mdl_governor_acquire(ctx->gov, host, jmin, jmax) == k_ra8_err_would_block) {
    return k_ra8_err_would_block;
  }
  size_t          got  = 0U;
  mdl_net_resp_t  resp = {};
  const ra8_err_t rc   = mdl_net_get_body(ctx->session->net, url, req, sink, &got, &resp);
  mdl_governor_observe(ctx->gov, host, resp.status, resp.retry_after);
  mdl_governor_release(ctx->gov, host);
  if (out_resp != nullptr) {
    *out_resp = resp;
  }
  if (out_bytes != nullptr) {
    *out_bytes = got;
  }
  return rc;
}

RA8_PRIV ra8_err_t priv_mdl_fetch_with_retry(mdl_fetch_ctx_t*     ctx,
                                             const char*          host,
                                             const char*          url,
                                             const mdl_net_req_t* req,
                                             mdl_net_body_sink_t* sink,
                                             uint32_t             jmin,
                                             uint32_t             jmax,
                                             mdl_net_resp_t*      out_resp,
                                             size_t*              out_bytes)
{
  ra8_err_t rc = k_ra8_fail;
  for (uint8_t attempt = 0U; attempt < (uint8_t)k_fetch_max_attempts; ++attempt) {
    rc = internal_mdl_fetch_governed_get_body(ctx,
                                              host,
                                              url,
                                              req,
                                              sink,
                                              jmin,
                                              jmax,
                                              out_resp,
                                              out_bytes);
    if (!priv_mdl_fetch_is_retryable(rc)) {
      break; /* success, or a permanent failure not worth retrying */
    }
  }
  return rc;
}

RA8_INTERNAL static ra8_err_t internal_fetch_asset_execute(mdl_fetch_ctx_t*     ctx,
                                                           const char*          host,
                                                           const char*          url,
                                                           const mdl_net_req_t* req,
                                                           uint32_t             jmin,
                                                           uint32_t             jmax,
                                                           const char*          target_abs,
                                                           mdl_net_resp_t*      out_resp,
                                                           size_t*              out_bytes)
{
  size_t              got  = 0U;
  mdl_net_resp_t      resp = {};
  mdl_fetch_body_t    body = {};
  ra8_err_t           rc   = priv_mdl_fetch_body_init_exact(&body, ctx->storage, target_abs);
  mdl_net_body_sink_t sink = priv_mdl_fetch_body_sink(&body);
  if (rc == k_ra8_ok) {
    rc = priv_mdl_fetch_with_retry(ctx, host, url, req, &sink, jmin, jmax, &resp, &got);
  }
  if (out_resp != nullptr) {
    *out_resp = resp;
  }
  if (rc == k_ra8_ok) {
    rc = priv_mdl_fetch_body_prepare(&body);
  }
  if (rc == k_ra8_ok) {
    rc = priv_mdl_fetch_body_commit(&body);
  }
  if (rc != k_ra8_ok) {
    const ra8_err_t aborted = priv_mdl_fetch_body_abort(&body);
    const ra8_err_t result  = (aborted == k_ra8_ok) ? rc : aborted;
    priv_mdl_fetch_record_fail(ctx, url, resp.status, result);
    return result;
  }
  if (out_bytes != nullptr) {
    *out_bytes = got;
  }
  return k_ra8_ok;
}
 
ra8_err_t mdl_fetch_asset(mdl_fetch_ctx_t* ctx,
                          const char*      url,
                          const char*      target_abs,
                          const char*      referer,
                          mdl_net_resp_t*  out_resp,
                          size_t*          out_bytes)
{
  if (out_resp != nullptr) {
    *out_resp = (mdl_net_resp_t){};
  }
  if (out_bytes != nullptr) {
    *out_bytes = 0U;
  }
  if ((ctx == nullptr) || (ctx->session == nullptr) || (ctx->session->net == nullptr) ||
      (ctx->site == nullptr) || (url == nullptr) || (url[0] == '\0') || (target_abs == nullptr) ||
      (target_abs[0] != '/')) {
    return k_ra8_err_invalid_arg;
  }
 
  uint32_t crawl = 0U;
  if (!mdl_session_url_allowed(ctx->session, url, &crawl)) {
    priv_mdl_fetch_record_fail(ctx, url, 0, k_ra8_fail);
    return k_ra8_fail;
  }
  char        hostbuf[k_mdl_gov_host_max];
  const char* host = internal_mdl_fetch_page_host(url, hostbuf, sizeof(hostbuf));
  if (host == nullptr) {
    return k_ra8_err_invalid_arg;
  }
 
  const uint32_t      jmin = internal_mdl_fetch_max_u32(ctx->site->img_delay_min, crawl);
  const uint32_t      jmax = internal_mdl_fetch_max_u32(ctx->site->img_delay_max, crawl);
  const mdl_net_req_t req  = {.user_agent        = ctx->session->user_agent,
                              .referer           = referer,
                              .if_none_match     = nullptr,
                              .if_modified_since = nullptr,
                              .timeout_ms        = ctx->timeout_ms};
  return internal_fetch_asset_execute(ctx,
                                      host,
                                      url,
                                      &req,
                                      jmin,
                                      jmax,
                                      target_abs,
                                      out_resp,
                                      out_bytes);
}

RA8_PRIV ra8_err_t priv_mdl_fetch_cache_get_buf(void*                context,
                                                const char*          url,
                                                const mdl_net_req_t* request,
                                                char*                buffer,
                                                size_t               capacity,
                                                size_t*              out_length,
                                                mdl_net_resp_t*      response)
{
  mdl_fetch_cache_request_t* fetch = (mdl_fetch_cache_request_t*)context;
  ra8_err_t                  error = k_ra8_fail;
  for (uint8_t attempt = 0U; attempt < (uint8_t)k_fetch_max_attempts; ++attempt) {
    error = mdl_governor_acquire(fetch->ctx->gov, fetch->host, fetch->jmin, fetch->jmax);
    if (error == k_ra8_ok) {
      error = mdl_net_get_buf(fetch->ctx->session->net,
                              url,
                              request,
                              buffer,
                              capacity,
                              out_length,
                              response);
      mdl_governor_observe(fetch->ctx->gov, fetch->host, response->status, response->retry_after);
      mdl_governor_release(fetch->ctx->gov, fetch->host);
    }
    if (!priv_mdl_fetch_is_retryable(error)) {
      break;
    }
  }
  return error;
}

RA8_INTERNAL static ra8_err_t
internal_mdl_fetch_chapter_html(mdl_fetch_ctx_t* ctx, const char* chapter_url, size_t* out_len)
{
  uint32_t crawl = 0U;
  if (!mdl_session_url_allowed(ctx->session, chapter_url, &crawl)) {
    priv_mdl_fetch_record_fail(ctx, chapter_url, 0, k_ra8_fail);
    return k_ra8_fail;
  }
  mdl_fetch_cache_request_t fetch = {
    .ctx  = ctx,
    .jmin = internal_mdl_fetch_max_u32(ctx->site->chapter_delay_min, crawl),
    .jmax = internal_mdl_fetch_max_u32(ctx->site->chapter_delay_max, crawl)};
  if (!mdl_url_host(chapter_url, fetch.host, sizeof(fetch.host))) {
    return k_ra8_err_invalid_arg;
  }
  const mdl_net_req_t req      = {.user_agent = ctx->session->user_agent,
                                  .referer    = ctx->series_url,
                                  .timeout_ms = ctx->timeout_ms};
  size_t              len      = 0U;
  mdl_net_resp_t      response = {};
  mdl_cache_result_t  cache_result;
  const ra8_err_t     rc = (ctx->cache != nullptr) ? mdl_cache_get_buf(ctx->cache,
                                                                       chapter_url,
                                                                       &req,
                                                                       priv_mdl_fetch_cache_get_buf,
                                                                       &fetch,
                                                                       ctx->page_buf,
                                                                       ctx->page_cap,
                                                                       &len,
                                                                       &response,
                                                                       &cache_result)
                                                   : priv_mdl_fetch_cache_get_buf(&fetch,
                                                                                  chapter_url,
                                                                                  &req,
                                                                                  ctx->page_buf,
                                                                                  ctx->page_cap,
                                                                                  &len,
                                                                                  &response);
  if (rc != k_ra8_ok) {
    priv_mdl_fetch_record_fail(ctx, chapter_url, response.status, rc);
    return k_ra8_fail;
  }
  const ra8_err_t erc = mdl_extract_images(ctx->page_buf,
                                           len,
                                           chapter_url,
                                           ctx->site->page_img_attr,
                                           ctx->site->page_img_url_contains,
                                           ctx->images);
  if (erc != k_ra8_ok) {
    priv_mdl_fetch_record_fail(ctx, chapter_url, response.status, erc);
    return erc;
  }
  if (ctx->images->count == 0U) {
    priv_mdl_fetch_record_fail(ctx, chapter_url, response.status, k_ra8_err_no_data);
    return k_ra8_err_no_data;
  }
  *out_len = len;
  return k_ra8_ok;
}

RA8_INTERNAL static bool internal_mdl_fetch_resolve_dest(mdl_fetch_ctx_t*   ctx,
                                                         mdl_fetch_layout_t layout,
                                                         const char*        id,
                                                         const char*        combined_abs,
                                                         const char*        combined_rel,
                                                         size_t             global_no,
                                                         char*              chap_abs,
                                                         size_t             chap_cap,
                                                         const char**       dest_abs,
                                                         const char**       dest_rel,
                                                         size_t*            base)
{
  if (layout == k_mdl_layout_combined) {
    *dest_abs = combined_abs;
    *dest_rel = combined_rel;
    *base     = global_no;
    return true;
  }
  if (!mdl_join_dir_under(ctx->storage, ctx->series_abs_dir, id, chap_abs, chap_cap)) {
    return false;
  }
  *dest_abs = chap_abs;
  *dest_rel = id;
  *base     = 0U;
  return true;
}

RA8_INTERNAL static void internal_mdl_fetch_mark_complete(mdl_fetch_ctx_t*   ctx,
                                                          mdl_fetch_layout_t layout,
                                                          mdl_chapter_rec_t* rec,
                                                          size_t*            global_no)
{
  rec->pages_done = (uint16_t)ctx->images->count;
  rec->complete   = true;
  rec->fetched_at = (int64_t)time(nullptr);
  if (layout == k_mdl_layout_combined) {
    *global_no += ctx->images->count;
  }
}

RA8_INTERNAL static bool internal_mdl_fetch_select_chapter_title(mdl_fetch_ctx_t*    ctx,
                                                                 const char*         chapter_url,
                                                                 mdl_chapter_work_t* work)
{
  work->title = work->rec->title;
  if (ctx->site->chapter_title_selector[0] == '\0') {
    if (work->rec->title[0] == '\0') {
      work->title = work->id;
    }
    return true;
  }
  const ra8_err_t rc = mdl_extract_selector(ctx->page_buf,
                                            work->html_len,
                                            ctx->site->chapter_title_selector,
                                            work->extracted_title,
                                            sizeof(work->extracted_title));
  if (rc == k_ra8_ok) {
    work->title = work->extracted_title;
    return true;
  }
  if (rc == k_ra8_err_invalid_size) {
    (void)internal_mdl_fetch_diag3(ctx,
                                   "mdl: chapter title for '",
                                   chapter_url,
                                   "' exceeds the persistent field; refusing truncation\n");
    priv_mdl_fetch_record_fail(ctx, chapter_url, 0, rc);
    return false;
  }
  const bool preserving   = work->rec->title[0] != '\0';
  ra8_err_t  output_error = internal_mdl_fetch_diag3(ctx,
                                                     "mdl: chapter title selector '",
                                                     ctx->site->chapter_title_selector,
                                                     "' failed for '");
  output_error            = priv_mdl_stream_text(output_error, ctx->diagnostic, chapter_url);
  output_error            = priv_mdl_stream_text(output_error, ctx->diagnostic, "'; ");
  output_error =
    priv_mdl_stream_text(output_error,
                         ctx->diagnostic,
                         preserving ? "preserving recorded title\n" : "using chapter id\n");
  if (output_error != k_ra8_ok) {
    ctx->progress_error = output_error;
    return false;
  }
  if (!preserving) {
    work->title = work->id;
  }
  return true;
}

RA8_INTERNAL static bool internal_mdl_fetch_select_chapter_number(mdl_fetch_ctx_t*    ctx,
                                                                  const char*         chapter_url,
                                                                  mdl_chapter_work_t* work)
{
  if (ctx->site->chapter_number_selector[0] == '\0') {
    return true;
  }
  char            extracted[k_mdl_title_max];
  const ra8_err_t rc = mdl_extract_selector(ctx->page_buf,
                                            work->html_len,
                                            ctx->site->chapter_number_selector,
                                            extracted,
                                            sizeof(extracted));
  if (rc == k_ra8_ok) {
    if (mdl_urlname_chapter_text_parse(extracted, &work->number)) {
      work->number_known = true;
      return true;
    }
    ra8_err_t output_error = internal_mdl_fetch_diag3(ctx,
                                                      "mdl: chapter number selector '",
                                                      ctx->site->chapter_number_selector,
                                                      "' returned invalid value '");
    output_error           = priv_mdl_stream_text(output_error, ctx->diagnostic, extracted);
    output_error           = priv_mdl_stream_text(output_error, ctx->diagnostic, "' for '");
    output_error           = priv_mdl_stream_text(output_error, ctx->diagnostic, chapter_url);
    output_error           = priv_mdl_stream_text(output_error, ctx->diagnostic, "'\n");
    if (output_error != k_ra8_ok) {
      ctx->progress_error = output_error;
    }
    priv_mdl_fetch_record_fail(ctx, chapter_url, 0, k_ra8_err_validation_failed);
    return false;
  }
  if (rc == k_ra8_err_invalid_size) {
    (void)internal_mdl_fetch_diag3(ctx,
                                   "mdl: chapter number for '",
                                   chapter_url,
                                   "' exceeds the bounded selector field\n");
    priv_mdl_fetch_record_fail(ctx, chapter_url, 0, rc);
    return false;
  }
  ra8_err_t output_error = internal_mdl_fetch_diag3(ctx,
                                                    "mdl: chapter number selector '",
                                                    ctx->site->chapter_number_selector,
                                                    "' failed for '");
  output_error           = priv_mdl_stream_text(output_error, ctx->diagnostic, chapter_url);
  output_error           = priv_mdl_stream_text(output_error, ctx->diagnostic, "'; preserving ");
  output_error =
    priv_mdl_stream_text(output_error,
                         ctx->diagnostic,
                         work->number_known ? "recorded number\n" : "unknown number\n");
  if (output_error != k_ra8_ok) {
    ctx->progress_error = output_error;
    return false;
  }
  return true;
}

RA8_INTERNAL static mdl_chap_status_t
internal_mdl_fetch_chapter_pages_and_checkpoint(mdl_fetch_ctx_t*    ctx,
                                                const char*         chapter_url,
                                                mdl_fetch_layout_t  layout,
                                                const char*         combined_abs,
                                                const char*         combined_rel,
                                                size_t*             global_no,
                                                mdl_fetch_stats_t*  stats,
                                                size_t              chapter_index,
                                                size_t              chapter_total,
                                                mdl_chapter_work_t* work)
{
  if (!mdl_state_set_chapter_metadata(work->rec, work->title, work->number, work->number_known)) {
    return k_ch_failed;
  }
  work->rec->page_count = (uint16_t)ctx->images->count;
  if (work->rec->pages_done > work->rec->page_count) {
    work->rec->pages_done = 0U;
  }
  char        chap_abs[PATH_MAX];
  const char* dest_abs = nullptr;
  const char* dest_rel = nullptr;
  size_t      base     = 0U;
  if (!internal_mdl_fetch_resolve_dest(ctx,
                                       layout,
                                       work->id,
                                       combined_abs,
                                       combined_rel,
                                       *global_no,
                                       chap_abs,
                                       sizeof(chap_abs),
                                       &dest_abs,
                                       &dest_rel,
                                       &base)) {
    return k_ch_failed;
  }
  const mdl_run_pos_t pos = {.chapter_index = chapter_index,
                             .chapter_total = chapter_total,
                             .chapter_id    = work->id};
  if (priv_mdl_fetch_chapter_pages(ctx,
                                   chapter_url,
                                   dest_abs,
                                   dest_rel,
                                   base,
                                   work->rec,
                                   stats,
                                   &pos) != k_ra8_ok) {
    (void)priv_mdl_fetch_checkpoint(ctx);
    return k_ch_failed;
  }
  internal_mdl_fetch_mark_complete(ctx, layout, work->rec, global_no);
  return (priv_mdl_fetch_checkpoint(ctx) == k_ra8_ok) ? k_ch_completed : k_ch_failed;
}

RA8_INTERNAL static mdl_chap_status_t internal_mdl_fetch_process_chapter(mdl_fetch_ctx_t* ctx,
                                                                         const char* chapter_url,
                                                                         mdl_fetch_layout_t layout,
                                                                         const char* combined_abs,
                                                                         const char* combined_rel,
                                                                         size_t*     global_no,
                                                                         mdl_fetch_stats_t* stats,
                                                                         size_t chapter_index,
                                                                         size_t chapter_total)
{
  mdl_chapter_work_t work = {};
  mdl_urlname_last_segment(chapter_url, work.id, sizeof(work.id));
  work.number_known = mdl_urlname_chapter_parse(chapter_url, &work.number);
  work.rec          = mdl_state_add_chapter_numbered(ctx->state,
                                                     work.id,
                                                     chapter_url,
                                                     work.number,
                                                     work.number_known);
  if (work.rec == nullptr) {
    return k_ch_failed;
  }
  work.number       = work.rec->number;
  work.number_known = work.rec->number_known;
  if (ctx->update_only && work.rec->complete) {
    *global_no += work.rec->page_count;
    return k_ch_skipped;
  }
  work.rec->complete   = false;
  work.rec->fetched_at = 0;
  if (internal_mdl_fetch_chapter_html(ctx, chapter_url, &work.html_len) != k_ra8_ok) {
    return k_ch_failed;
  }
  if (!internal_mdl_fetch_select_chapter_title(ctx, chapter_url, &work) ||
      !internal_mdl_fetch_select_chapter_number(ctx, chapter_url, &work)) {
    return k_ch_failed;
  }
  return internal_mdl_fetch_chapter_pages_and_checkpoint(ctx,
                                                         chapter_url,
                                                         layout,
                                                         combined_abs,
                                                         combined_rel,
                                                         global_no,
                                                         stats,
                                                         chapter_index,
                                                         chapter_total,
                                                         &work);
}

RA8_INTERNAL static bool internal_mdl_fetch_tally(mdl_chap_status_t s, mdl_fetch_stats_t* stats)
{
  if (s == k_ch_completed) {
    stats->chapters_completed += 1U;
  } else if (s == k_ch_skipped) {
    stats->chapters_skipped += 1U;
  } else {
    stats->chapters_failed += 1U;
  }
  return s == k_ch_failed;
}

RA8_INTERNAL static bool internal_mdl_fetch_ctx_ready(const mdl_fetch_ctx_t* ctx)
{
  if (ctx == nullptr) {
    return false;
  }
  return (ctx->session != nullptr) && (ctx->storage != nullptr) && (ctx->state != nullptr) &&
         (ctx->series_abs_dir != nullptr) && (ctx->series_url != nullptr) &&
         (ctx->site != nullptr) && (ctx->page_buf != nullptr) && (ctx->images != nullptr);
}
 
ra8_err_t mdl_fetch_run(mdl_fetch_ctx_t*      ctx,
                        const mdl_url_list_t* chapters,
                        mdl_fetch_layout_t    layout,
                        const char*           combined_dir_rel,
                        mdl_fetch_stats_t*    stats)
{
  if ((ctx == nullptr) || (chapters == nullptr) || (stats == nullptr)) {
    return k_ra8_err_invalid_arg;
  }
  if (!internal_mdl_fetch_ctx_ready(ctx)) {
    return k_ra8_err_invalid_arg;
  }
  memset(stats, 0, sizeof(*stats));
  ctx->progress_error = k_ra8_ok;
 
  char combined_abs[PATH_MAX];
  if (layout == k_mdl_layout_combined) {
    if ((combined_dir_rel == nullptr) || !mdl_join_dir_under(ctx->storage,
                                                             ctx->series_abs_dir,
                                                             combined_dir_rel,
                                                             combined_abs,
                                                             sizeof(combined_abs))) {
      return k_ra8_err_invalid_arg;
    }
  } else {
    combined_abs[0] = '\0';
  }
 
  size_t global_no = 0U;
  bool   any_fail  = false;
  for (size_t i = 0U; i < chapters->count; ++i) {
    const mdl_chap_status_t s = internal_mdl_fetch_process_chapter(ctx,
                                                                   chapters->urls[i],
                                                                   layout,
                                                                   combined_abs,
                                                                   combined_dir_rel,
                                                                   &global_no,
                                                                   stats,
                                                                   i + 1U,
                                                                   chapters->count);
    if (internal_mdl_fetch_tally(s, stats)) {
      any_fail = true;
      if ((ctx->progress_error != k_ra8_ok) || (layout == k_mdl_layout_combined)) {
        break; /* the combined archive cannot be completed past a gap */
      }
    }
  }
  if (ctx->progress_error != k_ra8_ok) {
    return ctx->progress_error;
  }
  return any_fail ? k_ra8_fail : k_ra8_ok;
}