mdl_app_prepare.c

Go to the documentation of this file.

#include "mdl_app_internal.h"

RA8_INTERNAL static ra8_err_t internal_prepare_cache_fetch(void*                context,
                                                           const char*          url,
                                                           const mdl_net_req_t* request,
                                                           char*                buffer,
                                                           size_t               capacity,
                                                           size_t*              out_length,
                                                           mdl_net_resp_t*      response)
{
  return mdl_net_get_buf((mdl_net_iface_t*)context,
                         url,
                         request,
                         buffer,
                         capacity,
                         out_length,
                         response);
}

RA8_PRIV ra8_err_t priv_mdl_app_report_cache(const char* url, const mdl_cache_result_t* result)
{
  ra8_io_stream_t* stream = priv_mdl_app_context()->diagnostic;
  ra8_err_t        error  = k_ra8_ok;
  if (result->index_rebuilt) {
    error = priv_mdl_stream_text(error, stream, "mdl: discarded corrupt cache index for ");
    error = priv_mdl_stream_text(error, stream, url);
    error = priv_mdl_stream_text(error, stream, "\n");
  }
  if ((error == k_ra8_ok) && result->body_reused) {
    error = priv_mdl_stream_text(error, stream, "mdl: cache reuse age=");
    if (result->age_seconds >= 0) {
      error = priv_mdl_stream_u64(error, stream, (uint64_t)result->age_seconds);
      error = priv_mdl_stream_text(error, stream, "s");
    } else {
      error = priv_mdl_stream_text(error, stream, "unknown");
    }
    error = priv_mdl_stream_text(error, stream, " status=");
    error = priv_mdl_stream_u64(error, stream, result->observed_status);
    error = priv_mdl_stream_text(error, stream, " url=");
    error = priv_mdl_stream_text(error, stream, url);
    error = priv_mdl_stream_text(error, stream, "\n");
  }
  return error;
}

static char s_rowtmp[k_mdl_url_max];

RA8_INTERNAL static ra8_err_t
internal_prepare_diag3(const char* first, const char* second, const char* third)
{
  ra8_io_stream_t* diagnostic = priv_mdl_app_context()->diagnostic;
  ra8_err_t        error      = priv_mdl_stream_text(k_ra8_ok, diagnostic, first);
  error                       = priv_mdl_stream_text(error, diagnostic, second);
  return priv_mdl_stream_text(error, diagnostic, third);
}

RA8_INTERNAL static void internal_swap_rows(mdl_url_list_t* l, size_t a, size_t b)
{
  if (a == b) {
    return;
  }
  memcpy(s_rowtmp, l->urls[a], k_mdl_url_max);
  memcpy(l->urls[a], l->urls[b], k_mdl_url_max);
  memcpy(l->urls[b], s_rowtmp, k_mdl_url_max);
}

RA8_INTERNAL static void internal_reverse_list(mdl_url_list_t* l)
{
  for (size_t i = 0U; i < (l->count / 2U); ++i) {
    internal_swap_rows(l, i, l->count - 1U - i);
  }
}

RA8_INTERNAL static bool internal_sort_by_chapter_num(mdl_url_list_t* l)
{
  for (size_t i = 0U; i < l->count; ++i) {
    double ignored = 0.0;
    if (!mdl_urlname_chapter_parse(l->urls[i], &ignored)) {
      return false;
    }
  }
  for (size_t i = 0U; i + 1U < l->count; ++i) {
    size_t min = i;
    for (size_t j = i + 1U; j < l->count; ++j) {
      if (mdl_urlname_chapter_value(l->urls[j]) < mdl_urlname_chapter_value(l->urls[min])) {
        min = j;
      }
    }
    internal_swap_rows(l, i, min);
  }
  return true;
}

RA8_INTERNAL static ra8_err_t internal_apply_order(mdl_url_list_t* l, mdl_chapter_order_t order)
{
  if (order == k_mdl_order_reverse) {
    internal_reverse_list(l);
  } else if (order == k_mdl_order_asc) {
    if (!internal_sort_by_chapter_num(l)) {
      return priv_mdl_stream_text(k_ra8_ok,
                                  priv_mdl_app_context()->diagnostic,
                                  "mdl: WARNING: at least one chapter has no numeric "
                                  "identifier; preserving site document order\n");
    }
  }
  return k_ra8_ok;
}

RA8_INTERNAL static void internal_filter_prefix(mdl_url_list_t* l, const char* prefix)
{
  const size_t plen = strlen(prefix);
  size_t       w    = 0U;
  for (size_t r = 0U; r < l->count; ++r) {
    if (strncmp(l->urls[r], prefix, plen) == 0) {
      if (w != r) {
        memcpy(l->urls[w], l->urls[r], k_mdl_url_max);
      }
      ++w;
    }
  }
  l->count = w;
}

RA8_INTERNAL static ra8_err_t internal_warn_no_contact(void)
{
  static bool s_warned = false;
  if (!s_warned) {
    const ra8_err_t error =
      priv_mdl_stream_text(k_ra8_ok,
                           priv_mdl_app_context()->diagnostic,
                           "mdl: WARNING: no contact configured; pass --contact "
                           "<email|url> so a site operator can reach you before banning\n");
    if (error == k_ra8_ok) {
      s_warned = true;
    }
    return error;
  }
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t
internal_extract_optional_metadata(const mdl_site_t* site, const char* html, size_t len)
{
  const struct {
    const char* selector; 
    char*       out;      
    size_t      cap;      
    const char* label;    
  } optional[] = {{site->series_summary_selector,
                   priv_mdl_app_context()->series_metadata.summary,
                   sizeof(priv_mdl_app_context()->series_metadata.summary),
                   "summary"},
                  {site->series_author_selector,
                   priv_mdl_app_context()->series_metadata.writer,
                   sizeof(priv_mdl_app_context()->series_metadata.writer),
                   "author"},
                  {site->series_artist_selector,
                   priv_mdl_app_context()->series_metadata.artist,
                   sizeof(priv_mdl_app_context()->series_metadata.artist),
                   "artist"}};
  for (size_t i = 0U; i < (sizeof(optional) / sizeof(optional[0])); ++i) {
    if (optional[i].selector[0] == '\0') {
      continue;
    }
    const ra8_err_t rc =
      mdl_extract_selector(html, len, optional[i].selector, optional[i].out, optional[i].cap);
    if (rc == k_ra8_err_invalid_size) {
      const ra8_err_t output_error =
        internal_prepare_diag3("mdl: series ",
                               optional[i].label,
                               " exceeds the bounded metadata field\n");
      return (output_error == k_ra8_ok) ? rc : output_error;
    }
    if (rc != k_ra8_ok) {
      optional[i].out[0] = '\0';
      ra8_err_t output_error =
        internal_prepare_diag3("mdl: WARNING: series ", optional[i].label, " selector '");
      output_error = priv_mdl_stream_text(output_error,
                                          priv_mdl_app_context()->diagnostic,
                                          optional[i].selector);
      output_error = priv_mdl_stream_text(output_error,
                                          priv_mdl_app_context()->diagnostic,
                                          "' matched no bounded value\n");
      if (output_error != k_ra8_ok) {
        return output_error;
      }
    }
  }
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t
internal_extract_cover(const mdl_site_t* site, const char* series_url, const char* html, size_t len)
{
  if (site->series_cover_selector[0] == '\0') {
    return k_ra8_ok;
  }
  char            cover_raw[k_mdl_url_max];
  const ra8_err_t rc =
    mdl_extract_selector(html, len, site->series_cover_selector, cover_raw, sizeof(cover_raw));
  if (rc != k_ra8_ok) {
    const ra8_err_t output_error = internal_prepare_diag3("mdl: required series cover selector '",
                                                          site->series_cover_selector,
                                                          "' failed\n");
    return (output_error == k_ra8_ok) ? rc : output_error;
  }
  if (!mdl_extract_resolve_url(series_url,
                               cover_raw,
                               priv_mdl_app_context()->series_metadata.cover_url,
                               sizeof(priv_mdl_app_context()->series_metadata.cover_url))) {
    const ra8_err_t output_error =
      priv_mdl_stream_text(k_ra8_ok,
                           priv_mdl_app_context()->diagnostic,
                           "mdl: series cover URL could not be resolved safely\n");
    return (output_error == k_ra8_ok) ? k_ra8_err_validation_failed : output_error;
  }
  return k_ra8_ok;
}

RA8_INTERNAL static ra8_err_t internal_extract_series_metadata(const mdl_site_t* site,
                                                               const char*       series_url,
                                                               const char*       html,
                                                               size_t            len)
{
  memset(&priv_mdl_app_context()->series_metadata,
         0,
         sizeof(priv_mdl_app_context()->series_metadata));
  priv_mdl_app_context()->series_metadata.direction =
    (strcmp(site->reading_direction, "rtl") == 0) ? k_mdl_state_read_rtl : k_mdl_state_read_ltr;
  const int language_len = snprintf(priv_mdl_app_context()->series_metadata.language,
                                    sizeof(priv_mdl_app_context()->series_metadata.language),
                                    "%s",
                                    site->language);
  if ((language_len < 0) ||
      ((size_t)language_len >= sizeof(priv_mdl_app_context()->series_metadata.language))) {
    return k_ra8_err_invalid_size;
  }
 
  ra8_err_t rc = k_ra8_err_no_data;
  if (site->series_title_selector[0] != '\0') {
    rc = mdl_extract_selector(html,
                              len,
                              site->series_title_selector,
                              priv_mdl_app_context()->series_metadata.title,
                              sizeof(priv_mdl_app_context()->series_metadata.title));
    priv_mdl_app_context()->series_metadata.title_selected = rc == k_ra8_ok;
  }
  if (rc == k_ra8_err_invalid_size) {
    const ra8_err_t output_error =
      priv_mdl_stream_text(k_ra8_ok,
                           priv_mdl_app_context()->diagnostic,
                           "mdl: series title exceeds the bounded metadata field\n");
    return (output_error == k_ra8_ok) ? rc : output_error;
  }
  if (rc != k_ra8_ok) {
    mdl_urlname_last_segment(series_url,
                             priv_mdl_app_context()->series_metadata.title,
                             sizeof(priv_mdl_app_context()->series_metadata.title));
    ra8_err_t output_error = internal_prepare_diag3("mdl: WARNING: series title selector '",
                                                    site->series_title_selector,
                                                    "' failed; using URL leaf '");
    output_error           = priv_mdl_stream_text(output_error,
                                                  priv_mdl_app_context()->diagnostic,
                                                  priv_mdl_app_context()->series_metadata.title);
    output_error = priv_mdl_stream_text(output_error, priv_mdl_app_context()->diagnostic, "'\n");
    if (output_error != k_ra8_ok) {
      return output_error;
    }
  }
 
  rc = internal_extract_optional_metadata(site, html, len);
  if (rc != k_ra8_ok) {
    return rc;
  }
  return internal_extract_cover(site, series_url, html, len);
}

RA8_INTERNAL static ra8_err_t internal_prepare_filter_chapters(const mdl_site_t* site,
                                                               const char*       series_url)
{
  char prefix[k_dir_path_bytes];
  if (site->chapter_url_prefix[0] != '\0') {
    const int prefix_len = snprintf(prefix, sizeof(prefix), "%s", site->chapter_url_prefix);
    if ((prefix_len < 0) || ((size_t)prefix_len >= sizeof(prefix))) {
      return k_ra8_err_invalid_size;
    }
  } else {
    const int prefix_len = snprintf(prefix, sizeof(prefix), "%s", series_url);
    if ((prefix_len < 0) || ((size_t)prefix_len >= sizeof(prefix))) {
      return k_ra8_err_invalid_size;
    }
    size_t pl = strlen(prefix);
    if ((pl > 0U) && (prefix[pl - 1U] != '/') && (pl + 1U < sizeof(prefix))) {
      prefix[pl]      = '/';
      prefix[pl + 1U] = '\0';
    }
  }
  internal_filter_prefix(&priv_mdl_app_context()->chapters, prefix);
  return internal_apply_order(&priv_mdl_app_context()->chapters, site->chapter_order);
}

RA8_PRIV ra8_err_t priv_mdl_app_prepare_chapters(const mdl_site_t* site,
                                                 const char*       series_url,
                                                 uint32_t          timeout,
                                                 mdl_cache_t*      cache)
{
  if ((cache == nullptr) ||
      !mdl_session_url_allowed(&priv_mdl_app_context()->session, series_url, nullptr)) {
    return k_ra8_fail; /* robots refused the series page (message printed) */
  }
  const mdl_net_req_t req      = {.user_agent = priv_mdl_app_context()->session.user_agent,
                                  .referer    = nullptr,
                                  .timeout_ms = timeout};
  size_t              len      = 0U;
  mdl_net_resp_t      response = {};
  mdl_cache_result_t  cache_result;
  ra8_err_t           rc = mdl_cache_get_buf(cache,
                                             series_url,
                                             &req,
                                             internal_prepare_cache_fetch,
                                             priv_mdl_app_context()->session.net,
                                             priv_mdl_app_context()->page,
                                             sizeof(priv_mdl_app_context()->page),
                                             &len,
                                             &response,
                                             &cache_result);
  if (rc != k_ra8_ok) {
    return rc;
  }
  rc = priv_mdl_app_report_cache(series_url, &cache_result);
  if (rc != k_ra8_ok) {
    return rc;
  }
  rc = internal_extract_series_metadata(site, series_url, priv_mdl_app_context()->page, len);
  if (rc != k_ra8_ok) {
    return rc;
  }
  rc = mdl_extract_anchors(priv_mdl_app_context()->page,
                           len,
                           series_url,
                           site->chapter_url_contains,
                           &priv_mdl_app_context()->chapters);
  if (rc != k_ra8_ok) {
    return rc;
  }
 
  return internal_prepare_filter_chapters(site, series_url);
}

RA8_PRIV ra8_err_t priv_mdl_app_start_session(mdl_net_iface_t*      net,
                                              const mdl_run_opts_t* opts,
                                              const char*           cfg_contact,
                                              char*                 ua,
                                              size_t                ua_cap)
{
  const char* contact = opts->contact;
  if ((contact == nullptr) && (cfg_contact != nullptr) && (cfg_contact[0] != '\0')) {
    contact = cfg_contact; /* CLI --contact overrides the site descriptor's key */
  }
  if (!mdl_session_build_ua(contact, ua, ua_cap)) {
    const ra8_err_t error = internal_warn_no_contact();
    if (error != k_ra8_ok) {
      return error;
    }
  }
  mdl_session_init(&priv_mdl_app_context()->session,
                   net,
                   ua,
                   priv_mdl_app_context()->diagnostic,
                   opts->honor_robots);
  return k_ra8_ok;
}

RA8_PRIV bool priv_mdl_app_prepare_series_dir(const char* out_dir,
                                              const char* series_url,
                                              char*       slug,
                                              size_t      slug_cap,
                                              char*       abs_dir)
{
  mdl_urlname_last_segment(series_url, slug, slug_cap);
  mdl_storage_t* storage = &priv_mdl_app_context()->storage;
  if (priv_mdl_app_storage_ensure_directory(storage, out_dir) != k_ra8_ok) {
    (void)internal_prepare_diag3("mdl: library root is not a directory: ", out_dir, "\n");
    return false;
  }
  if (!mdl_join_dir_under(storage, out_dir, slug, abs_dir, (size_t)k_fw_fs_path_cap)) {
    (void)internal_prepare_diag3("mdl: refusing unsafe series path for slug '", slug, "'\n");
    return false;
  }
  return true;
}